Initial commit

.envrc

@@ -0,0 +1,2 @@
+source_env ..
+use nix

README.md

@@ -0,0 +1,65 @@
+# video immutulator
+
+Rip from video silos into ERIS.
+
+It takes a bit of configuring. An example:
+```
+let ?ds = dataspace
+<eris $ds>
+<bind <ref {oid: "eris" key: #x""}> $ds #f>
+
+<daemon video_immutulator {
+  argv: ["/bin/video_immutulator"]
+  protocol: application/syndicate
+}>
+
+? <eris ?dataspace> $dataspace [
+
+  $dataspace ? <Observe <rec immutulation _ _ _ _> _> [
+    $config <require-service <daemon video_immutulator>>
+  ]
+
+  $dataspace ? <Observe <rec yt-dlp [<lit ?url> _]> _> [
+    let ?id = <yt-dlp $url>
+    $config <require-service <daemon $id>>
+    <daemon $id {
+        argv: [
+            "/bin/json_translator"
+            "yt-dlp"
+            "--embed-subs"
+            "--embed-metadata"
+            "--dump-json"
+            "--no-simulate"
+            "--output" "/tmp/%(id)s.%(ext)s"
+            $url
+          ]
+        protocol: application/syndicate
+      }>
+
+    $config ? <service-object <daemon $id> ?cap> [
+      $cap ? <recv ?json> [
+        $dataspace <yt-dlp $url $json>
+      ]
+    ]
+
+  ]
+
+  $config [
+    ? <service-object <daemon video_immutulator> ?cap> [
+      $cap {
+        cachefile: "/srv/immutulator.pr"
+        dataspace: $dataspace
+        stores: [ "coap+tcp://192.168.160.1:5683" ]
+      }
+    ]
+  ]
+]
+```
+
+And I use [syndump](https://git.syndicate-lang.org/ehmry/syndicate_utils#syndump) as a frontend:
+```
+$ SYNDICATE_STEP=$(mintsturdyref '"eris"' < /dev/null) syndump '<immutulation "https://www.youtube.com/watch?v=anwy2MPT5RE" ? ? ?>'
+
+# Output:
++ "urn:eris:B4A3BWB3NL3AG7JLQZJ3DEOH7G6FYG6LHFH2QTOV5VUEYTMK6NWTQEYVPP2JT2VN5YXXYLKKXR3MOF35DXFKTBOVTR55Z23JQO56D2S4MM" 6030153 video/webm
+```

Tuprules.tup

@@ -0,0 +1,7 @@
+include ../eris-nim/depends.tup
+include ../freedesktop_org/depends.tup
+include ../syndicate-nim/depends.tup
+NIM_FLAGS += --path:$(TUP_CWD)/../eris-nim/src
+NIM_FLAGS += --path:$(TUP_CWD)/../freedesktop_org/src
+NIM_FLAGS += --path:$(TUP_CWD)/../syndicate-nim/src
+NIM = nim

protocol.prs

@@ -0,0 +1,21 @@
+version 1.
+embeddedType EntityRef.Cap .
+
+Immutulation = <immutulation @source string @erisCap string @length int @mimetype symbol> .
+
+BootArgs = {
+  cachefile: string
+  dataspace: #!any
+  stores: [string ...]
+} .
+
+JSON =
+/ @string string
+/ @integer int
+/ @double double
+/ @boolean bool
+/ @null =null
+/ @array [JSON ...]
+/ @object { symbol: JSON ...:... } .
+
+YtDlp = <yt-dlp @url string @dump { symbol: JSON ...:... }> .

shell.nix

@@ -0,0 +1,10 @@
+let
+  syndicate = builtins.getFlake "syndicate";
+  pkgs =
+    import <nixpkgs> { overlays = (builtins.attrValues syndicate.overlays); };
+in pkgs.nimPackages.buildNimPackage (finalAttrs: prevAttrs: {
+  pname = "video_immutulator";
+  version = "unstable";
+  nativeBuildInputs = [ pkgs.pkg-config ];
+  propagatedBuildInputs = [ pkgs.nimPackages.getdns ];
+})

src/Tupfile

@@ -0,0 +1,4 @@
+include_rules
+: foreach ../*.prs |> !preserves_schema_nim |> {schema}
+: video_immutulator.nim | $(SYNDICATE_PROTOCOL) {schema} |> !nim_bin |> {bin}
+: {bin} |> !assert_built |>

src/protocol.nim

@@ -0,0 +1,57 @@
+
+import
+  preserves, std/tables
+
+type
+  YtDlp* {.acyclic, preservesRecord: "yt-dlp".} = ref object
+    `url`*: string
+    `dump`*: Table[Symbol, JSON]
+
+  BootArgs* {.preservesDictionary.} = object
+    `cachefile`*: string
+    `dataspace`* {.preservesEmbedded.}: Preserve[void]
+    `stores`*: seq[string]
+
+  JSONKind* {.pure.} = enum
+    `string`, `integer`, `double`, `boolean`, `null`, `array`, `object`
+  JSONString* = string
+  JSONInteger* = BiggestInt
+  JSONDouble* = float64
+  JSONBoolean* = bool
+  JSONArray* = seq[JSON]
+  JSONObject* = Table[Symbol, JSON]
+  `JSON`* {.acyclic, preservesOr.} = ref object
+    case orKind*: JSONKind
+    of JSONKind.`string`:
+        `string`*: JSONString
+
+    of JSONKind.`integer`:
+        `integer`*: JSONInteger
+
+    of JSONKind.`double`:
+        `double`*: JSONDouble
+
+    of JSONKind.`boolean`:
+        `boolean`*: JSONBoolean
+
+    of JSONKind.`null`:
+        `null`* {.preservesLiteral: "null".}: bool
+
+    of JSONKind.`array`:
+        `array`*: JSONArray
+
+    of JSONKind.`object`:
+        `object`*: JSONObject
+
+  
+  Immutulation* {.preservesRecord: "immutulation".} = object
+    `source`*: string
+    `erisCap`*: string
+    `length`*: BiggestInt
+    `mimetype`*: Symbol
+
+proc `$`*(x: YtDlp | BootArgs | JSON | Immutulation): string =
+  `$`(toPreserve(x))
+
+proc encode*(x: YtDlp | BootArgs | JSON | Immutulation): seq[byte] =
+  encode(toPreserve(x))

src/video_immutulator.nim

@@ -0,0 +1,57 @@
+# SPDX-FileCopyrightText: ☭ Emery Hemingway
+# SPDX-License-Identifier: Unlicense
+
+import std/[asyncdispatch, sequtils, os, streams, uri]
+import preserves, preserves/jsonhooks, syndicate
+from syndicate/protocols/dataspace import Observe
+import eris, eris/composite_stores, eris/url_stores
+import freedesktop_org
+import ./protocol
+
+type Observe = dataspace.Observe[Cap]
+
+proc immutulate(turn: var Turn; ds: Cap; store: ErisStore; source: string, cacheStream: Stream) =
+  onPublish(turn, ds, YtDlp ? {0: ?source, 1: grab()}) do (info: Preserve[void]):
+    let
+      filePath = info["filename".toSymbol].string
+      fileSize = filePath.getFileInfo.size
+      mimeTypes = mimeTypeOf(filePath)
+      fileStream = openFileStream(filePath)
+      chunkSize = recommendedChunkSize(fileSize)
+      ingest = newErisIngest(store, chunkSize, convergentMode)
+    var mimeType: string
+    if mimeTypes.len > 0:
+      mimeType = mimeTypes[0]
+    append(ingest, fileStream).addCallback(turn) do (turn: var Turn):
+      ingest.cap.addCallback(turn) do (turn: var Turn; ec: ErisCap):
+        var rec = initRecord(
+            "immutulation",
+            source.toPreserve(Cap),
+            ($ec).toPreserve(Cap),
+            fileSize.toPreserve(Cap),
+            mimeType.toSymbol(Cap),
+          )
+        discard publish(turn, ds, rec)
+        cacheStream.writeLine($rec)
+        cacheStream.flush()
+
+runActor("video_immutulator") do (root: Cap; turn: var Turn):
+  connectStdio(root, turn)
+  during(turn, root, ?BootArgs) do (cachefile: string, ds: Cap, stores: seq[string]):
+    let cacheStream = openFileStream(cachefile, fmAppend)
+
+    let store = newMultiStore()
+    if stores.len == 0: store.add(newDiscardStore())
+    else:
+      for s in stores:
+        let u = parseUri s
+        let c = waitFor newStoreClient(u)
+        stderr.writeLine "connected to ", u
+        store.add(c)
+
+    let pat = ?Observe(pattern: !Immutulation) ?? {0: grabLit()}
+    during(turn, ds, pat) do (source: string):
+      immutulate(turn, ds, store, source, cacheStream)
+
+  do:
+    close(store)

video_immutulator.nimble

@@ -0,0 +1,5 @@
+bin = @["video_immutulator"]
+license = "Unlicense"
+requires: "nim", "syndicate"
+srcDir = "src"
+version = "20230905"