Parallelize compilation
This commit is contained in:
parent
3b18ac6800
commit
4e62a7c95a
|
@ -18,8 +18,20 @@ case "$1" in
|
||||||
redo-ifchange ../syntax/all
|
redo-ifchange ../syntax/all
|
||||||
file=$(basename "$1")
|
file=$(basename "$1")
|
||||||
redo-ifchange "src/$file"
|
redo-ifchange "src/$file"
|
||||||
npx syndicate-babel "src/$file"
|
if [ -n "$SYNDICATE_COMPILE_SERVER" ]
|
||||||
# curl -fsS --data-binary @"src/$file" http://localhost:14641/compile/"$file"
|
then
|
||||||
|
if wget -q -O - --content-on-error --post-file="src/$file" \
|
||||||
|
${SYNDICATE_COMPILE_SERVER}/"$file" \
|
||||||
|
> ${targettempfile} 2>/dev/null
|
||||||
|
then
|
||||||
|
:
|
||||||
|
else
|
||||||
|
cat ${targettempfile} >&2
|
||||||
|
rm -f ${targettempfile}
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
npx syndicate-babel "src/$file"
|
||||||
|
fi
|
||||||
;;
|
;;
|
||||||
*/dist/*)
|
*/dist/*)
|
||||||
# Conservatively assume the distribution depends on ALL the
|
# Conservatively assume the distribution depends on ALL the
|
||||||
|
|
|
@ -1,4 +1,5 @@
|
||||||
#!/usr/bin/env node
|
#!/bin/sh
|
||||||
|
//bin/true; exec node --experimental-worker "$0"
|
||||||
const Program = require("../lib/index");
|
const Program = require("../lib/index");
|
||||||
const Syndicate = require("@syndicate-lang/core");
|
const Syndicate = require("@syndicate-lang/core");
|
||||||
Syndicate.bootModule(Program);
|
Syndicate.bootModule(Program);
|
||||||
|
|
|
@ -0,0 +1,30 @@
|
||||||
|
"use strict";
|
||||||
|
|
||||||
|
require("@syndicate-lang/syntax/lib/index"); // patches babel -- load before any of babel loads!!
|
||||||
|
const BabelTransform = require("@babel/core/lib/transform");
|
||||||
|
|
||||||
|
import { Observe, Dataspace, genUuid, Inbound, Outbound } from "@syndicate-lang/core";
|
||||||
|
|
||||||
|
import { WorkItem, JobResult, JobError } from "./job";
|
||||||
|
assertion type CompilationOptions(options);
|
||||||
|
assertion type Compilation(filename, input);
|
||||||
|
|
||||||
|
export {
|
||||||
|
CompilationOptions,
|
||||||
|
Compilation,
|
||||||
|
};
|
||||||
|
|
||||||
|
spawn named 'compiler' {
|
||||||
|
const worker = genUuid('worker');
|
||||||
|
during Inbound(CompilationOptions($options)) {
|
||||||
|
during Inbound(Observe(WorkItem(worker, Compilation($filename, $input), _))) {
|
||||||
|
const finalOptions = Object.assign({filename: '/' + filename}, options.toJS());
|
||||||
|
console.log(worker, 'compiling', filename, '...');
|
||||||
|
BabelTransform.transform(input, finalOptions, Dataspace.wrapExternal((err, output) => {
|
||||||
|
react assert Outbound(WorkItem(worker,
|
||||||
|
Compilation(filename, input),
|
||||||
|
err ? JobError(err.toString()) : JobResult(output.code)));
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -1,19 +1,13 @@
|
||||||
"use strict";
|
"use strict";
|
||||||
|
|
||||||
require("@syndicate-lang/syntax/lib/index"); // patches babel -- load before any of babel loads!!
|
import { Bytes, spawnWorker } from "@syndicate-lang/core";
|
||||||
const BabelTransform = require("@babel/core/lib/transform");
|
|
||||||
|
|
||||||
const UI = require("@syndicate-lang/driver-browser-ui");
|
|
||||||
// @jsx UI.html
|
|
||||||
// @jsxFrag UI.htmlFragment
|
|
||||||
|
|
||||||
const Http = activate require("@syndicate-lang/driver-http-node");
|
const Http = activate require("@syndicate-lang/driver-http-node");
|
||||||
const S = activate require("@syndicate-lang/driver-streams-node");
|
const S = activate require("@syndicate-lang/driver-streams-node");
|
||||||
|
const C = require("./compiler");
|
||||||
|
const J = activate require("./job");
|
||||||
|
|
||||||
const fs = require('fs');
|
const fs = require('fs');
|
||||||
|
|
||||||
import { Dataspace, Bytes, genUuid } from "@syndicate-lang/core";
|
|
||||||
|
|
||||||
const options = {
|
const options = {
|
||||||
"port": 14641,
|
"port": 14641,
|
||||||
"babel-options": JSON.parse(fs.readFileSync(__dirname + '/../.babelrc')),
|
"babel-options": JSON.parse(fs.readFileSync(__dirname + '/../.babelrc')),
|
||||||
|
@ -60,19 +54,21 @@ console.info(`http://localhost:${options.port}/compile/FILENAME`);
|
||||||
console.info(options);
|
console.info(options);
|
||||||
|
|
||||||
spawn named 'rootServer' {
|
spawn named 'rootServer' {
|
||||||
|
assert C.CompilationOptions(options["babel-options"]);
|
||||||
|
|
||||||
const server = Http.HttpServer(null, options.port);
|
const server = Http.HttpServer(null, options.port);
|
||||||
during Http.Request($reqId, server, 'post', ['compile', $file], _, $reqSeal) spawn named reqId {
|
during Http.Request($reqId, server, 'post', ['compile', $file], _, $reqSeal) spawn named reqId {
|
||||||
stop on retracted S.Readable(reqId);
|
stop on retracted S.Readable(reqId);
|
||||||
_collectSource.call(this, reqId, (source) => {
|
_collectSource.call(this, reqId, (source) => {
|
||||||
const finalOptions = Object.assign({filename: '/' + file}, options["babel-options"]);
|
react {
|
||||||
BabelTransform.transform(source, finalOptions, Dataspace.wrapExternal((err, output) => {
|
stop on asserted J.Job(C.Compilation(file, source), J.JobError($errmsg)) {
|
||||||
if (err) {
|
|
||||||
react assert Http.Response(
|
react assert Http.Response(
|
||||||
reqId, 400, "Error", {"Content-Type": "text/plain"}, err.toString());
|
reqId, 400, "Error", {"Content-Type": "text/plain"}, errmsg);
|
||||||
} else {
|
|
||||||
react assert Http.Response(reqId, 200, "OK", {}, output.code);
|
|
||||||
}
|
}
|
||||||
}));
|
stop on asserted J.Job(C.Compilation(file, source), J.JobResult($output)) {
|
||||||
|
react assert Http.Response(reqId, 200, "OK", {}, output);
|
||||||
|
}
|
||||||
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -85,3 +81,11 @@ function _collectSource(streamId, cb) {
|
||||||
cb(source);
|
cb(source);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
const nCPUs = require('os').cpus().length;
|
||||||
|
for (let i = 0; i < nCPUs; i++) {
|
||||||
|
// spawn dataspace activate C;
|
||||||
|
spawnWorker(__dirname + '/compiler.js');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
@ -0,0 +1,62 @@
|
||||||
|
const { currentFacet, Observe, List, Set } = require("@syndicate-lang/core");
|
||||||
|
|
||||||
|
assertion type WorkItem(worker, item, result);
|
||||||
|
assertion type Job(item, result);
|
||||||
|
assertion type JobResult(output);
|
||||||
|
assertion type JobError(err);
|
||||||
|
|
||||||
|
export {
|
||||||
|
WorkItem, Job,
|
||||||
|
JobResult, JobError,
|
||||||
|
};
|
||||||
|
|
||||||
|
spawn named 'JobDispatcher' {
|
||||||
|
field this.readyWorkers = Set();
|
||||||
|
field this.runningJobs = 0;
|
||||||
|
// dataflow {
|
||||||
|
// console.log(this.runningJobs, 'running jobs', this.readyWorkers.size, 'idle workers');
|
||||||
|
// }
|
||||||
|
|
||||||
|
on asserted Observe(Observe(WorkItem($w, _, _))) this.readyWorkers = this.readyWorkers.add(w);
|
||||||
|
on retracted Observe(Observe(WorkItem($w, _, _))) this.readyWorkers = this.readyWorkers.remove(w);
|
||||||
|
|
||||||
|
during Observe(Job($item, _)) {
|
||||||
|
on start this.runningJobs++;
|
||||||
|
on stop this.runningJobs--;
|
||||||
|
|
||||||
|
const waitForWorker = (retryCount) => {
|
||||||
|
if (retryCount === 3) {
|
||||||
|
console.error('Failed job, too many retries', item);
|
||||||
|
react {
|
||||||
|
assert Job(item, JobError(new Error("Too many retries")));
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
react {
|
||||||
|
const facet = currentFacet();
|
||||||
|
dataflow {
|
||||||
|
const worker = this.readyWorkers.first();
|
||||||
|
if (worker !== void 0) {
|
||||||
|
this.readyWorkers = this.readyWorkers.remove(worker);
|
||||||
|
facet.stop(() => {
|
||||||
|
react {
|
||||||
|
stop on retracted Observe(Observe(WorkItem(worker, _, _))) {
|
||||||
|
console.warn('Worker withdrew before answering', worker);
|
||||||
|
waitForWorker(retryCount + 1);
|
||||||
|
}
|
||||||
|
stop on asserted WorkItem(worker, item, $result) {
|
||||||
|
this.readyWorkers = this.readyWorkers.add(worker);
|
||||||
|
react {
|
||||||
|
assert Job(item, result);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
on start waitForWorker(0);
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in New Issue