Arbitrary immutulation, not just videos

2023-09-05 23:44:26 +02:00 · 2023-09-05 23:44:26 +02:00 · ca3500cdad
parent a8c13b8e72
commit ca3500cdad
7 changed files with 55 additions and 87 deletions
--- a/README.md
+++ b/README.md
@ -1,6 +1,6 @@
-# video immutulator
+# The Immutulator

-Rip from video silos into ERIS.
+Ingest files into ERIS.

 It takes a bit of configuring. An example:
 ```
@ -8,48 +8,61 @@ let ?ds = dataspace
 <eris $ds>
 <bind <ref {oid: "eris" key: #x""}> $ds #f>

-<daemon video_immutulator {
-  argv: ["/bin/video_immutulator"]
+<daemon immutulator {
+  argv: ["/bin/immutulator"]
  protocol: application/syndicate
 }>

 ? <eris ?dataspace> $dataspace [

-  $dataspace ? <Observe <rec immutulation _ _ _ _> _> [
-    $config <require-service <daemon video_immutulator>>
+  ; require the daemon when immutulating
+  $dataspace ? <Observe <rec immutulation _> _> [
+    $config <require-service <daemon immutulator>>
  ]

-  $dataspace ? <Observe <rec yt-dlp [<lit ?url> _]> _> [
+  ; resolve  <immutulation-yt-dlp …> from <immutulation …>
+  $dataspace ? <Observe <rec immutulation-yt-dlp [<lit ?url> _ _ _]> _> [
+
+    ; temporarily start yt-dlp
    let ?id = <yt-dlp $url>
    $config <require-service <daemon $id>>
-    <daemon $id {
-        argv: [
-            "/bin/json_translator"
-            "yt-dlp"
-            "--embed-subs"
-            "--embed-metadata"
-            "--dump-json"
-            "--no-simulate"
-            "--output" "/tmp/%(id)s.%(ext)s"
-            $url
-          ]
-        protocol: application/syndicate
-      }>
+    ; translate the output of yt-dlp --dump-json
+    $config ? <built json_translator ?translator _> [
+      <daemon $id {
+          argv: [
+              $translator
+              "yt-dlp"
+              "--embed-subs"
+              "--embed-metadata"
+              "--dump-json"
+              "--no-simulate"
+              "--output" "/tmp/%(id)s.%(ext)s"
+              $url
+            ]
+          protocol: application/syndicate
+        }>
+    ]

    $config ? <service-object <daemon $id> ?cap> [
-      $cap ? <recv ?json> [
-        $dataspace <yt-dlp $url $json>
+      ; grap the filename out of the received JSON data
+      $cap ? <recv {filename: ?filename}> [
+        $config <require-service <daemon immutulator>>
+        ; ask the real immutulator
+        $dataspace ? <immutulation $filename ?eris ?size ?mime> [
+          ; answer the initial observation
+          $dataspace <immutulation-yt-dlp $url $eris $size $mime>
+        ]
      ]
    ]

  ]

  $config [
-    ? <service-object <daemon video_immutulator> ?cap> [
+    ? <service-object <daemon immutulator> ?cap> [
      $cap {
        cachefile: "/srv/immutulator.pr"
        dataspace: $dataspace
-        stores: [ "coap+tcp://192.168.160.1:5683" ]
+        stores: [ "coap+tcp://[::1]:5683" ]
      }
    ]
  ]
@ -58,7 +71,7 @@ let ?ds = dataspace

 And I use [syndump](https://git.syndicate-lang.org/ehmry/syndicate_utils#syndump) as a frontend:
 ```
-$ SYNDICATE_STEP=$(mintsturdyref '"eris"' < /dev/null) syndump '<immutulation "https://www.youtube.com/watch?v=anwy2MPT5RE" ? ? ?>'
+$ SYNDICATE_STEP=$(mintsturdyref '"eris"' < /dev/null) syndump '<immutulation-yt-dlp "https://www.youtube.com/watch?v=anwy2MPT5RE" ? ? ?>'

 # Output:
 + "urn:eris:B4A3BWB3NL3AG7JLQZJ3DEOH7G6FYG6LHFH2QTOV5VUEYTMK6NWTQEYVPP2JT2VN5YXXYLKKXR3MOF35DXFKTBOVTR55Z23JQO56D2S4MM" 6030153 video/webm
--- a/video_immutulator.nimble
+++ b/video_immutulator.nimble
@ -1,4 +1,4 @@
-bin = @["video_immutulator"]
+bin = @["immutulator"]
 license = "Unlicense"
 requires: "nim", "syndicate"
 srcDir = "src"
--- a/protocol.prs
+++ b/protocol.prs
@ -1,21 +1,10 @@
 version 1.
 embeddedType EntityRef.Cap .

-Immutulation = <immutulation @source string @erisCap string @length int @mimetype symbol> .
+Immutulation = <immutulation @filePath string @erisCap string @length int @mimetype symbol> .

 BootArgs = {
  cachefile: string
  dataspace: #!any
  stores: [string ...]
 } .
-
-JSON =
-/ @string string
-/ @integer int
-/ @double double
-/ @boolean bool
-/ @null =null
-/ @array [JSON ...]
-/ @object { symbol: JSON ...:... } .
-
-YtDlp = <yt-dlp @url string @dump { symbol: JSON ...:... }> .
--- a/shell.nix
+++ b/shell.nix
@ -3,7 +3,7 @@ let
  pkgs =
    import <nixpkgs> { overlays = (builtins.attrValues syndicate.overlays); };
 in pkgs.nimPackages.buildNimPackage (finalAttrs: prevAttrs: {
-  pname = "video_immutulator";
+  pname = "immutulator";
  version = "unstable";
  nativeBuildInputs = [ pkgs.pkg-config ];
  propagatedBuildInputs = [ pkgs.nimPackages.getdns ];
--- a/src/Tupfile
+++ b/src/Tupfile
@ -1,4 +1,4 @@
 include_rules
 : foreach ../*.prs |> !preserves_schema_nim |> {schema}
-: video_immutulator.nim | $(SYNDICATE_PROTOCOL) {schema} |> !nim_bin |> {bin}
+: immutulator.nim | $(SYNDICATE_PROTOCOL) {schema} |> !nim_bin |> {bin}
 : {bin} |> !assert_built |>
--- a/src/video_immutulator.nim
+++ b/src/video_immutulator.nim
@ -10,10 +10,10 @@ import ./protocol

 type Observe = dataspace.Observe[Cap]

-proc immutulate(turn: var Turn; ds: Cap; store: ErisStore; source: string, cacheStream: Stream) =
-  onPublish(turn, ds, YtDlp ? {0: ?source, 1: grab()}) do (info: Preserve[void]):
+proc immutulate(turn: var Turn; ds: Cap; store: ErisStore; filePath: string, cacheStream: Stream) =
+  if fileExists(filePath):
+    stderr.writeLine "immutulate ", filePath
    let
-      filePath = info["filename".toSymbol].string
      fileSize = filePath.getFileInfo.size
      mimeTypes = mimeTypeOf(filePath)
      fileStream = openFileStream(filePath)
@ -26,7 +26,7 @@ proc immutulate(turn: var Turn; ds: Cap; store: ErisStore; source: string, cache
      ingest.cap.addCallback(turn) do (turn: var Turn; ec: ErisCap):
        var rec = initRecord(
            "immutulation",
-            source.toPreserve(Cap),
+            filePath.toPreserve(Cap),
            ($ec).toPreserve(Cap),
            fileSize.toPreserve(Cap),
            mimeType.toSymbol(Cap),
@ -34,8 +34,10 @@ proc immutulate(turn: var Turn; ds: Cap; store: ErisStore; source: string, cache
        discard publish(turn, ds, rec)
        cacheStream.writeLine($rec)
        cacheStream.flush()
+  else:
+    stderr.writeLine "cannot immutulate file that does not exist: ", filePath

-runActor("video_immutulator") do (root: Cap; turn: var Turn):
+runActor("immutulator") do (root: Cap; turn: var Turn):
  connectStdio(root, turn)
  during(turn, root, ?BootArgs) do (cachefile: string, ds: Cap, stores: seq[string]):
    let cacheStream = openFileStream(cachefile, fmAppend)
@ -50,8 +52,8 @@ runActor("video_immutulator") do (root: Cap; turn: var Turn):
        store.add(c)

    let pat = ?Observe(pattern: !Immutulation) ?? {0: grabLit()}
-    during(turn, ds, pat) do (source: string):
-      immutulate(turn, ds, store, source, cacheStream)
+    during(turn, ds, pat) do (filePath: string):
+      immutulate(turn, ds, store, filePath, cacheStream)

  do:
    close(store)
--- a/src/protocol.nim
+++ b/src/protocol.nim
@ -1,57 +1,21 @@

 import
-  preserves, std/tables
+  preserves

 type
-  YtDlp* {.acyclic, preservesRecord: "yt-dlp".} = ref object
-    `url`*: string
-    `dump`*: Table[Symbol, JSON]
-
  BootArgs* {.preservesDictionary.} = object
    `cachefile`*: string
    `dataspace`* {.preservesEmbedded.}: Preserve[void]
    `stores`*: seq[string]

-  JSONKind* {.pure.} = enum
-    `string`, `integer`, `double`, `boolean`, `null`, `array`, `object`
-  JSONString* = string
-  JSONInteger* = BiggestInt
-  JSONDouble* = float64
-  JSONBoolean* = bool
-  JSONArray* = seq[JSON]
-  JSONObject* = Table[Symbol, JSON]
-  `JSON`* {.acyclic, preservesOr.} = ref object
-    case orKind*: JSONKind
-    of JSONKind.`string`:
-        `string`*: JSONString
-
-    of JSONKind.`integer`:
-        `integer`*: JSONInteger
-
-    of JSONKind.`double`:
-        `double`*: JSONDouble
-
-    of JSONKind.`boolean`:
-        `boolean`*: JSONBoolean
-
-    of JSONKind.`null`:
-        `null`* {.preservesLiteral: "null".}: bool
-
-    of JSONKind.`array`:
-        `array`*: JSONArray
-
-    of JSONKind.`object`:
-        `object`*: JSONObject
-
-  
  Immutulation* {.preservesRecord: "immutulation".} = object
-    `source`*: string
+    `filePath`*: string
    `erisCap`*: string
    `length`*: BiggestInt
    `mimetype`*: Symbol

-proc `$`*(x: YtDlp | BootArgs | JSON | Immutulation): string =
+proc `$`*(x: BootArgs | Immutulation): string =
  `$`(toPreserve(x))

-proc encode*(x: YtDlp | BootArgs | JSON | Immutulation): seq[byte] =
+proc encode*(x: BootArgs | Immutulation): seq[byte] =
  encode(toPreserve(x))