diff --git a/README.md b/README.md index b2ee60c..160e332 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# video immutulator +# The Immutulator -Rip from video silos into ERIS. +Ingest files into ERIS. It takes a bit of configuring. An example: ``` @@ -8,48 +8,61 @@ let ?ds = dataspace $ds #f> - ? $dataspace [ - $dataspace ? _> [ - $config > + ; require the daemon when immutulating + $dataspace ? _> [ + $config > ] - $dataspace ? _]> _> [ + ; resolve from + $dataspace ? _ _ _]> _> [ + + ; temporarily start yt-dlp let ?id = $config > - + ; translate the output of yt-dlp --dump-json + $config ? [ + + ] $config ? ?cap> [ - $cap ? [ - $dataspace + ; grap the filename out of the received JSON data + $cap ? [ + $config > + ; ask the real immutulator + $dataspace ? [ + ; answer the initial observation + $dataspace + ] ] ] ] $config [ - ? ?cap> [ + ? ?cap> [ $cap { cachefile: "/srv/immutulator.pr" dataspace: $dataspace - stores: [ "coap+tcp://192.168.160.1:5683" ] + stores: [ "coap+tcp://[::1]:5683" ] } ] ] @@ -58,7 +71,7 @@ let ?ds = dataspace And I use [syndump](https://git.syndicate-lang.org/ehmry/syndicate_utils#syndump) as a frontend: ``` -$ SYNDICATE_STEP=$(mintsturdyref '"eris"' < /dev/null) syndump '' +$ SYNDICATE_STEP=$(mintsturdyref '"eris"' < /dev/null) syndump '' # Output: + "urn:eris:B4A3BWB3NL3AG7JLQZJ3DEOH7G6FYG6LHFH2QTOV5VUEYTMK6NWTQEYVPP2JT2VN5YXXYLKKXR3MOF35DXFKTBOVTR55Z23JQO56D2S4MM" 6030153 video/webm diff --git a/video_immutulator.nimble b/immutulator.nimble similarity index 75% rename from video_immutulator.nimble rename to immutulator.nimble index 7160462..f5183b0 100644 --- a/video_immutulator.nimble +++ b/immutulator.nimble @@ -1,4 +1,4 @@ -bin = @["video_immutulator"] +bin = @["immutulator"] license = "Unlicense" requires: "nim", "syndicate" srcDir = "src" diff --git a/protocol.prs b/protocol.prs index 89fb352..5c5a1ca 100644 --- a/protocol.prs +++ b/protocol.prs @@ -1,21 +1,10 @@ version 1. embeddedType EntityRef.Cap . -Immutulation = . +Immutulation = . BootArgs = { cachefile: string dataspace: #!any stores: [string ...] } . - -JSON = -/ @string string -/ @integer int -/ @double double -/ @boolean bool -/ @null =null -/ @array [JSON ...] -/ @object { symbol: JSON ...:... } . - -YtDlp = . diff --git a/shell.nix b/shell.nix index 76ada28..e3460a1 100644 --- a/shell.nix +++ b/shell.nix @@ -3,7 +3,7 @@ let pkgs = import { overlays = (builtins.attrValues syndicate.overlays); }; in pkgs.nimPackages.buildNimPackage (finalAttrs: prevAttrs: { - pname = "video_immutulator"; + pname = "immutulator"; version = "unstable"; nativeBuildInputs = [ pkgs.pkg-config ]; propagatedBuildInputs = [ pkgs.nimPackages.getdns ]; diff --git a/src/Tupfile b/src/Tupfile index e42b53e..efdf7fe 100644 --- a/src/Tupfile +++ b/src/Tupfile @@ -1,4 +1,4 @@ include_rules : foreach ../*.prs |> !preserves_schema_nim |> {schema} -: video_immutulator.nim | $(SYNDICATE_PROTOCOL) {schema} |> !nim_bin |> {bin} +: immutulator.nim | $(SYNDICATE_PROTOCOL) {schema} |> !nim_bin |> {bin} : {bin} |> !assert_built |> diff --git a/src/video_immutulator.nim b/src/immutulator.nim similarity index 78% rename from src/video_immutulator.nim rename to src/immutulator.nim index ac3c52d..4bd93eb 100644 --- a/src/video_immutulator.nim +++ b/src/immutulator.nim @@ -10,10 +10,10 @@ import ./protocol type Observe = dataspace.Observe[Cap] -proc immutulate(turn: var Turn; ds: Cap; store: ErisStore; source: string, cacheStream: Stream) = - onPublish(turn, ds, YtDlp ? {0: ?source, 1: grab()}) do (info: Preserve[void]): +proc immutulate(turn: var Turn; ds: Cap; store: ErisStore; filePath: string, cacheStream: Stream) = + if fileExists(filePath): + stderr.writeLine "immutulate ", filePath let - filePath = info["filename".toSymbol].string fileSize = filePath.getFileInfo.size mimeTypes = mimeTypeOf(filePath) fileStream = openFileStream(filePath) @@ -26,7 +26,7 @@ proc immutulate(turn: var Turn; ds: Cap; store: ErisStore; source: string, cache ingest.cap.addCallback(turn) do (turn: var Turn; ec: ErisCap): var rec = initRecord( "immutulation", - source.toPreserve(Cap), + filePath.toPreserve(Cap), ($ec).toPreserve(Cap), fileSize.toPreserve(Cap), mimeType.toSymbol(Cap), @@ -34,8 +34,10 @@ proc immutulate(turn: var Turn; ds: Cap; store: ErisStore; source: string, cache discard publish(turn, ds, rec) cacheStream.writeLine($rec) cacheStream.flush() + else: + stderr.writeLine "cannot immutulate file that does not exist: ", filePath -runActor("video_immutulator") do (root: Cap; turn: var Turn): +runActor("immutulator") do (root: Cap; turn: var Turn): connectStdio(root, turn) during(turn, root, ?BootArgs) do (cachefile: string, ds: Cap, stores: seq[string]): let cacheStream = openFileStream(cachefile, fmAppend) @@ -50,8 +52,8 @@ runActor("video_immutulator") do (root: Cap; turn: var Turn): store.add(c) let pat = ?Observe(pattern: !Immutulation) ?? {0: grabLit()} - during(turn, ds, pat) do (source: string): - immutulate(turn, ds, store, source, cacheStream) + during(turn, ds, pat) do (filePath: string): + immutulate(turn, ds, store, filePath, cacheStream) do: close(store) diff --git a/src/protocol.nim b/src/protocol.nim index 829300d..c8eb64f 100644 --- a/src/protocol.nim +++ b/src/protocol.nim @@ -1,57 +1,21 @@ import - preserves, std/tables + preserves type - YtDlp* {.acyclic, preservesRecord: "yt-dlp".} = ref object - `url`*: string - `dump`*: Table[Symbol, JSON] - BootArgs* {.preservesDictionary.} = object `cachefile`*: string `dataspace`* {.preservesEmbedded.}: Preserve[void] `stores`*: seq[string] - JSONKind* {.pure.} = enum - `string`, `integer`, `double`, `boolean`, `null`, `array`, `object` - JSONString* = string - JSONInteger* = BiggestInt - JSONDouble* = float64 - JSONBoolean* = bool - JSONArray* = seq[JSON] - JSONObject* = Table[Symbol, JSON] - `JSON`* {.acyclic, preservesOr.} = ref object - case orKind*: JSONKind - of JSONKind.`string`: - `string`*: JSONString - - of JSONKind.`integer`: - `integer`*: JSONInteger - - of JSONKind.`double`: - `double`*: JSONDouble - - of JSONKind.`boolean`: - `boolean`*: JSONBoolean - - of JSONKind.`null`: - `null`* {.preservesLiteral: "null".}: bool - - of JSONKind.`array`: - `array`*: JSONArray - - of JSONKind.`object`: - `object`*: JSONObject - - Immutulation* {.preservesRecord: "immutulation".} = object - `source`*: string + `filePath`*: string `erisCap`*: string `length`*: BiggestInt `mimetype`*: Symbol -proc `$`*(x: YtDlp | BootArgs | JSON | Immutulation): string = +proc `$`*(x: BootArgs | Immutulation): string = `$`(toPreserve(x)) -proc encode*(x: YtDlp | BootArgs | JSON | Immutulation): seq[byte] = +proc encode*(x: BootArgs | Immutulation): seq[byte] = encode(toPreserve(x))