replace some dataflow with internal events in typed flink
This commit is contained in:
parent
5f38b6cc94
commit
8f8f4c416f
|
@ -219,7 +219,7 @@ The JobManager then performs the job and, when finished, asserts (job-finished I
|
||||||
(define (spawn-task-manager)
|
(define (spawn-task-manager)
|
||||||
(define id (gensym 'task-manager))
|
(define id (gensym 'task-manager))
|
||||||
(spawn τc
|
(spawn τc
|
||||||
(print-role
|
(begin
|
||||||
(start-facet tm
|
(start-facet tm
|
||||||
(log "Task Manager (TM) ~a is running" id)
|
(log "Task Manager (TM) ~a is running" id)
|
||||||
(during (job-manager-alive)
|
(during (job-manager-alive)
|
||||||
|
@ -320,9 +320,12 @@ The JobManager then performs the job and, when finished, asserts (job-finished I
|
||||||
[(task $id (reduce-work $l $r))
|
[(task $id (reduce-work $l $r))
|
||||||
(task id (reduce-work (left l) (left r)))]))
|
(task id (reduce-work (left l) (left r)))]))
|
||||||
|
|
||||||
|
(assertion-struct assigned-task : SelectedTM (mngr))
|
||||||
|
(message-struct tasks-finished : TasksFinished (id))
|
||||||
|
|
||||||
(define (spawn-job-manager)
|
(define (spawn-job-manager)
|
||||||
(spawn τc
|
(spawn τc
|
||||||
(begin
|
(print-role
|
||||||
(start-facet jm
|
(start-facet jm
|
||||||
(assert (job-manager-alive))
|
(assert (job-manager-alive))
|
||||||
(log "Job Manager Up")
|
(log "Job Manager Up")
|
||||||
|
@ -375,6 +378,7 @@ The JobManager then performs the job and, when finished, asserts (job-finished I
|
||||||
(empty? (ref ready-tasks))
|
(empty? (ref ready-tasks))
|
||||||
(empty? (ref waiting-tasks)))
|
(empty? (ref waiting-tasks)))
|
||||||
(log "JM finished with job ~a" job-id)
|
(log "JM finished with job ~a" job-id)
|
||||||
|
(realize! (tasks-finished job-id))
|
||||||
(start-facet done (assert (job-finished job-id data)))]
|
(start-facet done (assert (job-finished job-id data)))]
|
||||||
[else
|
[else
|
||||||
;; TODO - in MapReduce, there should be either 1 waiting task, or 0, meaning the job is done.
|
;; TODO - in MapReduce, there should be either 1 waiting task, or 0, meaning the job is done.
|
||||||
|
@ -408,10 +412,10 @@ The JobManager then performs the job and, when finished, asserts (job-finished I
|
||||||
(define (assign-task [mngr : ID])
|
(define (assign-task [mngr : ID])
|
||||||
(start-facet assign
|
(start-facet assign
|
||||||
(assert (task-assignment mngr job-id t))
|
(assert (task-assignment mngr job-id t))
|
||||||
(on (retracted (task-manager mngr discard))
|
(know (assigned-task mngr))
|
||||||
|
(on (retracted (task-manager mngr _))
|
||||||
;; our task manager has crashed
|
;; our task manager has crashed
|
||||||
(stop assign
|
(stop assign))
|
||||||
(set! task-mngr not-a-real-task-manager)))
|
|
||||||
(on start
|
(on start
|
||||||
;; N.B. when this line was here, and not after `(when mngr ...)` above,
|
;; N.B. when this line was here, and not after `(when mngr ...)` above,
|
||||||
;; things didn't work. I think that due to script scheduling, all ready
|
;; things didn't work. I think that due to script scheduling, all ready
|
||||||
|
@ -430,30 +434,43 @@ The JobManager then performs the job and, when finished, asserts (job-finished I
|
||||||
;; don't think we need a release-slot! here, because if we've heard back from a task manager,
|
;; don't think we need a release-slot! here, because if we've heard back from a task manager,
|
||||||
;; they should have told us a different slot count since we tried to give them work
|
;; they should have told us a different slot count since we tried to give them work
|
||||||
(log "JM overloaded manager ~a with task ~a" mngr this-id)
|
(log "JM overloaded manager ~a with task ~a" mngr this-id)
|
||||||
(stop assign
|
(stop assign)]
|
||||||
(set! task-mngr not-a-real-task-manager))]
|
|
||||||
[(finished $results)
|
[(finished $results)
|
||||||
(log "JM receives the results of task ~a" this-id)
|
(log "JM receives the results of task ~a" this-id)
|
||||||
(stop perform (k this-id results))]))))
|
(stop perform (k this-id results))]))))
|
||||||
|
|
||||||
(define (select-a-task-manager)
|
(define (select-a-task-manager)
|
||||||
(start-facet select
|
(start-facet select
|
||||||
(begin/dataflow
|
|
||||||
(when (equal? (ref task-mngr) not-a-real-task-manager)
|
(field [mngr (Maybe ID) none])
|
||||||
|
|
||||||
|
(define (try-assign!)
|
||||||
(define mngr?
|
(define mngr?
|
||||||
(for/first ([(id slots) (ref task-managers)]
|
(for/first ([(id slots) (ref task-managers)]
|
||||||
#:when (positive? (- slots (hash-ref/failure (ref requests-in-flight) id 0))))
|
#:when (positive? (- slots (hash-ref/failure (ref requests-in-flight) id 0))))
|
||||||
id))
|
id))
|
||||||
(match mngr?
|
(match mngr?
|
||||||
[(some $mngr)
|
[(some $m)
|
||||||
(take-slot! mngr)
|
(take-slot! m)
|
||||||
(set! task-mngr mngr)
|
(set! mngr (some m))
|
||||||
(assign-task mngr)]
|
(assign-task m)]
|
||||||
[none
|
[none
|
||||||
#f])))))
|
#f]))
|
||||||
|
|
||||||
|
(begin/dataflow
|
||||||
|
(when (equal? (ref mngr) none)
|
||||||
|
(try-assign!)))
|
||||||
|
|
||||||
|
(on (forget (assigned-task $m:ID))
|
||||||
|
(when (equal? (some m) (ref mngr))
|
||||||
|
(set! mngr none)))))
|
||||||
|
|
||||||
(on start (select-a-task-manager))))
|
(on start (select-a-task-manager))))
|
||||||
|
|
||||||
|
(on start
|
||||||
|
(start-facet delegate-tasks
|
||||||
|
(on (realize (tasks-finished job-id))
|
||||||
|
(stop delegate-tasks))
|
||||||
(begin/dataflow
|
(begin/dataflow
|
||||||
(define slots (slots-available))
|
(define slots (slots-available))
|
||||||
(define-tuple (ts readys)
|
(define-tuple (ts readys)
|
||||||
|
@ -462,7 +479,7 @@ The JobManager then performs the job and, when finished, asserts (job-finished I
|
||||||
(perform-task t push-results))
|
(perform-task t push-results))
|
||||||
(unless (empty? ts)
|
(unless (empty? ts)
|
||||||
;; the empty? check may be necessary to avoid a dataflow loop
|
;; the empty? check may be necessary to avoid a dataflow loop
|
||||||
(set! ready-tasks readys))))))))
|
(set! ready-tasks readys))))))))))
|
||||||
|
|
||||||
;; ---------------------------------------------------------------------------------------------------
|
;; ---------------------------------------------------------------------------------------------------
|
||||||
;; Client
|
;; Client
|
||||||
|
@ -492,5 +509,5 @@ The JobManager then performs the job and, when finished, asserts (job-finished I
|
||||||
(spawn-task-manager)
|
(spawn-task-manager)
|
||||||
(spawn-task-runner)
|
(spawn-task-runner)
|
||||||
(spawn-task-runner)
|
(spawn-task-runner)
|
||||||
#;(spawn-client (file->job "lorem.txt"))
|
(spawn-client (file->job "lorem.txt"))
|
||||||
(spawn-client (string->job INPUT)))
|
(spawn-client (string->job INPUT)))
|
||||||
|
|
Loading…
Reference in New Issue