Arbitrary immutulation, not just videos

README.md

@@ -1,6 +1,6 @@
-# video immutulator
+# The Immutulator
 
-Rip from video silos into ERIS.
+Ingest files into ERIS.
 
 It takes a bit of configuring. An example:
 ```
@@ -8,48 +8,61 @@ let ?ds = dataspace
 <eris $ds>
 <bind <ref {oid: "eris" key: #x""}> $ds #f>
 
-<daemon video_immutulator {
+<daemon immutulator {
-  argv: ["/bin/video_immutulator"]
+  argv: ["/bin/immutulator"]
   protocol: application/syndicate
 }>
 
 ? <eris ?dataspace> $dataspace [
 
-  $dataspace ? <Observe <rec immutulation _ _ _ _> _> [
+  ; require the daemon when immutulating
-    $config <require-service <daemon video_immutulator>>
+  $dataspace ? <Observe <rec immutulation _> _> [
+    $config <require-service <daemon immutulator>>
   ]
 
-  $dataspace ? <Observe <rec yt-dlp [<lit ?url> _]> _> [
+  ; resolve  <immutulation-yt-dlp …> from <immutulation …>
+  $dataspace ? <Observe <rec immutulation-yt-dlp [<lit ?url> _ _ _]> _> [
+
+    ; temporarily start yt-dlp
     let ?id = <yt-dlp $url>
     $config <require-service <daemon $id>>
-    <daemon $id {
+    ; translate the output of yt-dlp --dump-json
-        argv: [
+    $config ? <built json_translator ?translator _> [
-            "/bin/json_translator"
+      <daemon $id {
-            "yt-dlp"
+          argv: [
-            "--embed-subs"
+              $translator
-            "--embed-metadata"
+              "yt-dlp"
-            "--dump-json"
+              "--embed-subs"
-            "--no-simulate"
+              "--embed-metadata"
-            "--output" "/tmp/%(id)s.%(ext)s"
+              "--dump-json"
-            $url
+              "--no-simulate"
-          ]
+              "--output" "/tmp/%(id)s.%(ext)s"
-        protocol: application/syndicate
+              $url
-      }>
+            ]
+          protocol: application/syndicate
+        }>
+    ]
 
     $config ? <service-object <daemon $id> ?cap> [
-      $cap ? <recv ?json> [
+      ; grap the filename out of the received JSON data
-        $dataspace <yt-dlp $url $json>
+      $cap ? <recv {filename: ?filename}> [
+        $config <require-service <daemon immutulator>>
+        ; ask the real immutulator
+        $dataspace ? <immutulation $filename ?eris ?size ?mime> [
+          ; answer the initial observation
+          $dataspace <immutulation-yt-dlp $url $eris $size $mime>
+        ]
       ]
     ]
 
   ]
 
   $config [
-    ? <service-object <daemon video_immutulator> ?cap> [
+    ? <service-object <daemon immutulator> ?cap> [
       $cap {
         cachefile: "/srv/immutulator.pr"
         dataspace: $dataspace
-        stores: [ "coap+tcp://192.168.160.1:5683" ]
+        stores: [ "coap+tcp://[::1]:5683" ]
       }
     ]
   ]
@@ -58,7 +71,7 @@ let ?ds = dataspace
 
 And I use [syndump](https://git.syndicate-lang.org/ehmry/syndicate_utils#syndump) as a frontend:
 ```
-$ SYNDICATE_STEP=$(mintsturdyref '"eris"' < /dev/null) syndump '<immutulation "https://www.youtube.com/watch?v=anwy2MPT5RE" ? ? ?>'
+$ SYNDICATE_STEP=$(mintsturdyref '"eris"' < /dev/null) syndump '<immutulation-yt-dlp "https://www.youtube.com/watch?v=anwy2MPT5RE" ? ? ?>'
 
 # Output:
 + "urn:eris:B4A3BWB3NL3AG7JLQZJ3DEOH7G6FYG6LHFH2QTOV5VUEYTMK6NWTQEYVPP2JT2VN5YXXYLKKXR3MOF35DXFKTBOVTR55Z23JQO56D2S4MM" 6030153 video/webm

immutulator.nimble

@@ -1,4 +1,4 @@
-bin = @["video_immutulator"]
+bin = @["immutulator"]
 license = "Unlicense"
 requires: "nim", "syndicate"
 srcDir = "src"

protocol.prs

@@ -1,21 +1,10 @@
 version 1.
 embeddedType EntityRef.Cap .
 
-Immutulation = <immutulation @source string @erisCap string @length int @mimetype symbol> .
+Immutulation = <immutulation @filePath string @erisCap string @length int @mimetype symbol> .
 
 BootArgs = {
   cachefile: string
   dataspace: #!any
   stores: [string ...]
 } .
-
-JSON =
-/ @string string
-/ @integer int
-/ @double double
-/ @boolean bool
-/ @null =null
-/ @array [JSON ...]
-/ @object { symbol: JSON ...:... } .
-
-YtDlp = <yt-dlp @url string @dump { symbol: JSON ...:... }> .

shell.nix

@@ -3,7 +3,7 @@ let
   pkgs =
     import <nixpkgs> { overlays = (builtins.attrValues syndicate.overlays); };
 in pkgs.nimPackages.buildNimPackage (finalAttrs: prevAttrs: {
-  pname = "video_immutulator";
+  pname = "immutulator";
   version = "unstable";
   nativeBuildInputs = [ pkgs.pkg-config ];
   propagatedBuildInputs = [ pkgs.nimPackages.getdns ];

src/Tupfile

@@ -1,4 +1,4 @@
 include_rules
 : foreach ../*.prs |> !preserves_schema_nim |> {schema}
-: video_immutulator.nim | $(SYNDICATE_PROTOCOL) {schema} |> !nim_bin |> {bin}
+: immutulator.nim | $(SYNDICATE_PROTOCOL) {schema} |> !nim_bin |> {bin}
 : {bin} |> !assert_built |>

src/immutulator.nim

@@ -10,10 +10,10 @@ import ./protocol
 
 type Observe = dataspace.Observe[Cap]
 
-proc immutulate(turn: var Turn; ds: Cap; store: ErisStore; source: string, cacheStream: Stream) =
+proc immutulate(turn: var Turn; ds: Cap; store: ErisStore; filePath: string, cacheStream: Stream) =
-  onPublish(turn, ds, YtDlp ? {0: ?source, 1: grab()}) do (info: Preserve[void]):
+  if fileExists(filePath):
+    stderr.writeLine "immutulate ", filePath
     let
-      filePath = info["filename".toSymbol].string
       fileSize = filePath.getFileInfo.size
       mimeTypes = mimeTypeOf(filePath)
       fileStream = openFileStream(filePath)
@@ -26,7 +26,7 @@ proc immutulate(turn: var Turn; ds: Cap; store: ErisStore; source: string, cache
       ingest.cap.addCallback(turn) do (turn: var Turn; ec: ErisCap):
         var rec = initRecord(
             "immutulation",
-            source.toPreserve(Cap),
+            filePath.toPreserve(Cap),
             ($ec).toPreserve(Cap),
             fileSize.toPreserve(Cap),
             mimeType.toSymbol(Cap),
@@ -34,8 +34,10 @@ proc immutulate(turn: var Turn; ds: Cap; store: ErisStore; source: string, cache
         discard publish(turn, ds, rec)
         cacheStream.writeLine($rec)
         cacheStream.flush()
+  else:
+    stderr.writeLine "cannot immutulate file that does not exist: ", filePath
 
-runActor("video_immutulator") do (root: Cap; turn: var Turn):
+runActor("immutulator") do (root: Cap; turn: var Turn):
   connectStdio(root, turn)
   during(turn, root, ?BootArgs) do (cachefile: string, ds: Cap, stores: seq[string]):
     let cacheStream = openFileStream(cachefile, fmAppend)
@@ -50,8 +52,8 @@ runActor("video_immutulator") do (root: Cap; turn: var Turn):
         store.add(c)
 
     let pat = ?Observe(pattern: !Immutulation) ?? {0: grabLit()}
-    during(turn, ds, pat) do (source: string):
+    during(turn, ds, pat) do (filePath: string):
-      immutulate(turn, ds, store, source, cacheStream)
+      immutulate(turn, ds, store, filePath, cacheStream)
 
   do:
     close(store)

src/protocol.nim

@@ -1,57 +1,21 @@
 
 import
-  preserves, std/tables
+  preserves
 
 type
-  YtDlp* {.acyclic, preservesRecord: "yt-dlp".} = ref object
-    `url`*: string
-    `dump`*: Table[Symbol, JSON]
-
   BootArgs* {.preservesDictionary.} = object
     `cachefile`*: string
     `dataspace`* {.preservesEmbedded.}: Preserve[void]
     `stores`*: seq[string]
 
-  JSONKind* {.pure.} = enum
-    `string`, `integer`, `double`, `boolean`, `null`, `array`, `object`
-  JSONString* = string
-  JSONInteger* = BiggestInt
-  JSONDouble* = float64
-  JSONBoolean* = bool
-  JSONArray* = seq[JSON]
-  JSONObject* = Table[Symbol, JSON]
-  `JSON`* {.acyclic, preservesOr.} = ref object
-    case orKind*: JSONKind
-    of JSONKind.`string`:
-        `string`*: JSONString
-
-    of JSONKind.`integer`:
-        `integer`*: JSONInteger
-
-    of JSONKind.`double`:
-        `double`*: JSONDouble
-
-    of JSONKind.`boolean`:
-        `boolean`*: JSONBoolean
-
-    of JSONKind.`null`:
-        `null`* {.preservesLiteral: "null".}: bool
-
-    of JSONKind.`array`:
-        `array`*: JSONArray
-
-    of JSONKind.`object`:
-        `object`*: JSONObject
-
-  
   Immutulation* {.preservesRecord: "immutulation".} = object
-    `source`*: string
+    `filePath`*: string
     `erisCap`*: string
     `length`*: BiggestInt
     `mimetype`*: Symbol
 
-proc `$`*(x: YtDlp | BootArgs | JSON | Immutulation): string =
+proc `$`*(x: BootArgs | Immutulation): string =
   `$`(toPreserve(x))
 
-proc encode*(x: YtDlp | BootArgs | JSON | Immutulation): seq[byte] =
+proc encode*(x: BootArgs | Immutulation): seq[byte] =
   encode(toPreserve(x))