diff --git a/.github/workflows/IntegrationTest.yml b/.github/workflows/IntegrationTest.yml index 6120da58..265dabb5 100644 --- a/.github/workflows/IntegrationTest.yml +++ b/.github/workflows/IntegrationTest.yml @@ -14,8 +14,8 @@ jobs: fail-fast: false matrix: package: - - {user: TuringLang, repo: AdvancedPS.jl} - - {user: TuringLang, repo: Turing.jl} + - {user: TuringLang, repo: AdvancedPS.jl, ref: taped-libtask} + - {user: TuringLang, repo: Turing.jl, ref: master} steps: - uses: actions/checkout@v2 @@ -28,6 +28,7 @@ jobs: uses: actions/checkout@v2 with: repository: ${{ matrix.package.user }}/${{ matrix.package.repo }} + ref: ${{ matrix.package.ref }} path: downstream - name: Load this and run the downstream tests shell: julia --color=yes --project=downstream {0} diff --git a/.github/workflows/MemLayoutGen.yaml b/.github/workflows/MemLayoutGen.yaml deleted file mode 100644 index c233d8f5..00000000 --- a/.github/workflows/MemLayoutGen.yaml +++ /dev/null @@ -1,42 +0,0 @@ -name: MemLayout Info Generating -on: - pull_request: -jobs: - test: - runs-on: ${{ matrix.os }} - continue-on-error: true - strategy: - matrix: - version: - - '1.3.1' - - '1.4.2' - - '1.5.3' - - '1.5.4' - - '1.6.1' - os: - - ubuntu-latest - - windows-latest - - macOS-latest - arch: - - x64 - - x86 - exclude: - - os: macOS-latest - arch: x86 - steps: - - name: Set up MinGW - uses: egor-tensin/setup-mingw@v2 - with: - platform: i686 - cc: 1 - if: ${{ matrix.os == 'windows-latest' && matrix.arch == 'x86' }} - - uses: actions/checkout@v2 - - uses: julia-actions/setup-julia@v1 - with: - version: ${{ matrix.version }} - arch: ${{ matrix.arch }} - - run: julia .github/workflows/tasklayout/memlayout.jl - - uses: EndBug/add-and-commit@v7 - with: - add: 'src/memlayout' - author_name: GitHub Actions diff --git a/.github/workflows/Testing.yaml b/.github/workflows/Testing.yaml index fa17a364..5a62d5a0 100644 --- a/.github/workflows/Testing.yaml +++ b/.github/workflows/Testing.yaml @@ -41,7 +41,5 @@ jobs: ${{ runner.os }}-test-${{ env.cache-name }}- ${{ runner.os }}-test- ${{ runner.os }}- - - if: ${{ matrix.version == 'nightly' }} - run: julia --color=yes --project -e 'using Pkg; Pkg.add(PackageSpec(url="https://github.com/JuliaBinaryWrappers/Libtask_jll.jl.git"))' - uses: julia-actions/julia-buildpkg@latest - uses: julia-actions/julia-runtest@latest diff --git a/.github/workflows/tasklayout/memlayout.cpp b/.github/workflows/tasklayout/memlayout.cpp deleted file mode 100644 index 6004eb9e..00000000 --- a/.github/workflows/tasklayout/memlayout.cpp +++ /dev/null @@ -1,138 +0,0 @@ -/** - * linux64: g++ -I $JULIA_HOME/include/julia/ memlayout.cpp -o memlayout - * linux32: apt-get install g++-multilib - * g++ -march=pentium4 -m32 -I $JULIA_HOME/include/julia/ memlayout.cpp -o memlayout - * macOS: g++ -std=c++11 -I $JULIA_APP/Contents/Resources/julia/include/julia/ memlayout.cpp -o memlayout - * win64: g++ -I $JULIA_HOME/include/julia/ memlayout.cpp -o memlayout - * win32: g++ -I $JULIA_HOME/include/julia/ memlayout.cpp -o memlayout - **/ - -#include -#include -#include -#include - -#include - -#include "julia.h" - -std::string jl_ver(std::string sep=".") { - std::ostringstream oss; - oss << JULIA_VERSION_MAJOR << sep - << JULIA_VERSION_MINOR << sep - << JULIA_VERSION_PATCH; - return oss.str(); -} - -std::string platform() { -#ifdef _WIN32 -#ifdef _WIN64 - return "windows-x86_64"; -#endif - return "windows-x86"; -#endif -#ifdef __APPLE__ - return "darwin-x86_64"; -#endif -#ifdef __linux__ -#ifdef __x86_64 - return "linux-x86_64"; -#else - return "linux-x86"; -#endif -#endif -} - -std::map task_field_offsets() { - std::map data; - -#define field_info(f) data[#f] = offsetof(jl_task_t, f) - - field_info(next); // 131 142 153 154 160 170 - field_info(queue); // 131 142 153 154 160 170 - field_info(tls); // 131 142 153 154 160 170 - field_info(donenotify); // 131 142 153 154 160 170 - field_info(result); // 131 142 153 154 160 170 - -#if JULIA_VERSION_MAJOR == 1 && JULIA_VERSION_MINOR < 6 - field_info(state); // 131 142 153 154 - field_info(exception); // 131 142 153 154 - field_info(backtrace); // 131 142 153 154 -#endif - - field_info(logstate); // 131 142 153 154 160 170 - field_info(start); // 131 142 153 154 160 170 - field_info(sticky); // 131 142 153 154 160 170 - -#if JULIA_VERSION_MAJOR == 1 && JULIA_VERSION_MINOR > 5 - field_info(_state); // 160 170 - field_info(_isexception); // 160 170 -#endif - -#if JULIA_VERSION_MAJOR == 1 && JULIA_VERSION_MINOR > 6 - field_info(rngState0); // 170 - field_info(rngState1); // 170 - field_info(rngState2); // 170 - field_info(rngState3); // 170 -#endif - - // hidden state - field_info(gcstack); // 131 142 153 154 160 170 - -#if JULIA_VERSION_MAJOR == 1 && \ - (JULIA_VERSION_MINOR > 6 || JULIA_VERSION_MINOR < 5 || \ - (JULIA_VERSION_MINOR == 5 && JULIA_VERSION_PATCH < 4)) - field_info(world_age); // 131 142 153 170 -#endif - -#if JULIA_VERSION_MAJOR == 1 && JULIA_VERSION_MINOR > 6 - field_info(ptls); // 170 -#endif - field_info(tid); // 131 142 153 154 160 170 - field_info(prio); // 131 142 153 154 160 170 - field_info(excstack); // 131 142 153 154 160 170 - field_info(eh); // 131 142 153 154 160 170 - field_info(ctx); // 131 142 153 154 160 170 - -#if JULIA_VERSION_MAJOR == 1 && JULIA_VERSION_MINOR < 6 - field_info(locks); // 131 142 153 154 - field_info(timing_stack); // 131 142 153 154 -#endif - // field_info(copy_stack_ctx); - -#if defined(JL_TSAN_ENABLED) - field_info(tsan_state); // 170 -#endif - - field_info(stkbuf); // 131 142 153 154 160 170 - field_info(bufsz); // 131 142 153 154 160 170 - // field_info(copy_stack); - // field_info(started); - -#undef field_info - data["copy_stack"] = offsetof(jl_task_t, bufsz) + sizeof(size_t); - data["sizeof_ctx"] = sizeof(((jl_task_t*)0)->ctx); - // data["sizeof_stack_ctx"] = sizeof(((jl_task_t*)0)->copy_stack_ctx); - data["tls_base_context"] = offsetof(struct _jl_tls_states_t, base_ctx); - // data["tls_copy_stack_ctx"] = offsetof(struct _jl_tls_states_t, copy_stack_ctx); - return data; -} - -int main() { - std::ofstream ofs(platform() +"-v" + jl_ver("_") + ".jl", std::ofstream::out); - - ofs << "ALL_TASK_OFFSETS[(" - << '"' << platform() << '"' << ", " - << "v\"" << jl_ver() <<"\"" - <<")] = Dict(\n"; - ofs << " :END => " << sizeof(jl_task_t) << ",\n"; - - std::map data = task_field_offsets(); - for(auto kv: data) { - ofs << " :" << kv.first << " => " << kv.second << ",\n"; - } - ofs << ")\n"; - - ofs.close(); - return 0; -} diff --git a/.github/workflows/tasklayout/memlayout.jl b/.github/workflows/tasklayout/memlayout.jl deleted file mode 100644 index 78ec9ae5..00000000 --- a/.github/workflows/tasklayout/memlayout.jl +++ /dev/null @@ -1,32 +0,0 @@ -const ARCH = @static if string(Sys.ARCH)[1] == 'i' - "x86" -else - string(Sys.ARCH) -end - -const PLATFORM = @static if Sys.islinux() - "linux-" * ARCH -elseif Sys.iswindows() - "windows-" * ARCH -elseif Sys.isapple() - "darwin-" * ARCH -end - -OUTPUT = "$(PLATFORM)-v$(VERSION.major)_$(VERSION.minor)_$(VERSION.patch).jl" -OUTPUT_DEST = joinpath("src/memlayout", OUTPUT) - -isfile(OUTPUT_DEST) && exit(0) - -const PROJECT_DIR = (@__DIR__) |> dirname |> dirname |> dirname -const INCLUDE = joinpath(dirname(Sys.BINDIR), "include/julia") -const OPTIONS = if string(Sys.ARCH)[1] == 'i' - Sys.islinux() && run(`sudo apt-get install g++-multilib -y`) - ["-std=c++11","-march=pentium4", "-m32", "-static-libgcc", "-static-libstdc++"] -else - ["-std=c++11"] -end - -run(`g++ $(OPTIONS) -I$(INCLUDE) $(PROJECT_DIR)/.github/workflows/tasklayout/memlayout.cpp -o memlayout-gen.exe`) -run(`./memlayout-gen.exe`) - -isfile(OUTPUT) && Base.Filesystem.mv(OUTPUT, OUTPUT_DEST) diff --git a/Project.toml b/Project.toml index 9ee5aa26..b84ecef1 100644 --- a/Project.toml +++ b/Project.toml @@ -3,16 +3,15 @@ uuid = "6f1fad26-d15e-5dc8-ae53-837a1d7b8c9f" license = "MIT" desc = "C shim for task copying in Turing" repo = "https://github.com/TuringLang/Libtask.jl.git" -version = "0.5.3" +version = "0.6" [deps] -Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" -Libtask_jll = "3ae2931a-708c-5973-9c38-ccf7496fb450" +IRTools = "7869d1d1-7146-5819-86e3-90919afe41df" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +MacroTools = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" [compat] -Libtask_jll = "0.5.1" julia = "1.3" [extras] diff --git a/src/Libtask.jl b/src/Libtask.jl index 22897916..12a8c516 100644 --- a/src/Libtask.jl +++ b/src/Libtask.jl @@ -1,21 +1,19 @@ module Libtask -using Libdl -using Libtask_jll +using IRTools +using MacroTools -export CTask, consume, produce, TArray, tzeros, tfill, TRef +export CTask, consume, produce +export TArray, tzeros, tfill, TRef -function __init__() - @static if VERSION < v"1.6.0" - push!(Libdl.DL_LOAD_PATH, - Libtask_jll.get_libtask_julia_path() |> dirname) - Libdl.dlopen(Libtask_jll.libtask_julia_path) - end -end +export TapedTask + +include("tapedfunction.jl") +include("tapedtask.jl") -include("memlayout/main.jl") -include("ctask.jl") include("tarray.jl") include("tref.jl") +CTask = TapedTask + end diff --git a/src/ctask.jl b/src/ctask.jl deleted file mode 100644 index a3e6a754..00000000 --- a/src/ctask.jl +++ /dev/null @@ -1,280 +0,0 @@ -""" - CTask - -Wrapper of a [`Task`](@ref) for which deep copying of stack allocated objects is enabled. -""" -struct CTask - task::Task - - function CTask(task::Task) - ret = new(enable_stack_copying(task)) - task.storage === nothing && (task.storage = IdDict()) - task.storage[:ctask] = ret - ret - end -end - -CTask(f) = CTask(Task(task_wrapper(f))) - -# Iteration interface. -Base.iterate(ctask::CTask, state=nothing) = consume(ctask), nothing -Base.IteratorSize(::Type{CTask}) = Base.SizeUnknown() -Base.IteratorEltype(::Type{CTask}) = Base.EltypeUnknown() - -struct CTaskException <: Exception - task::Task -end - -function Base.showerror(io::IO, ex::CTaskException) - println(io, "CTaskException:") - ct = ex.task - bt = @static if VERSION < v"1.6.0-DEV.1145" - ct.backtrace - else - ct.storage[:_libtask_bt] - end - showerror(io, ct.exception, bt) -end - -# Utility function for self-copying mechanism -_current_task() = ccall((:vanilla_get_current_task, libtask_julia), Ref{Task}, ()) - -n_copies() = n_copies(_current_task()) -function n_copies(t::Task) - t.storage === nothing && (t.storage = IdDict()) - return get!(t.storage, :n_copies, 0) -end - -function enable_stack_copying(t::Task) - if istaskfailed(t) - error("only runnable or finished tasks' stack can be copied.") - end - # return ccall((:jl_enable_stack_copying, libtask_julia), - # Any, (Any,), t)::Task - copy_stack = internal_getfield(t, :copy_stack, Int32) - if copy_stack == 0 - internal_setfield(t, :copy_stack, Int32(1)) - internal_setfield(t, :bufsz, Csize_t(0)) - ccall((:jl_reset_task_ctx, libtask_julia), - Cvoid, (Any, Csize_t, Csize_t, Csize_t), - t, TASK_OFFSETS[:ctx], TASK_OFFSETS[:sizeof_ctx], - TASK_OFFSETS[:tls_base_context]) - end - return t -end - -""" - task_wrapper() - -`task_wrapper` is a wordaround for set the result/exception to the -correct task which maybe copied/forked from another one(the original -one). Without this, the result/exception is always sent to the -original task. That is done in `JULIA_PROJECT/src/task.c`, the -function `start_task` and `finish_task`. - -This workaround is not the proper way to do the work it does. The -proper way is refreshing the `current_task` (the variable `t`) in -`start_task` after the call to `jl_apply` returns. - -""" -function task_wrapper(func) - f = let func=func - () -> begin - try - ct = _current_task() - res = func() - ct.result = res - ct.storage === nothing && (ct.storage = IdDict()) - ct.storage[:_libtask_state] = :done - wait() - catch ex - ct = _current_task() - @static if VERSION < v"1.6.0-DEV.1145" - ct.exception = ex - ct.backtrace = catch_backtrace() - else - ct._isexception = true - ct.storage[:_libtask_bt] = catch_backtrace() - end - ct.result = ex - ct.storage === nothing && (ct.storage = IdDict()) - ct.storage[:_libtask_state] = :failed - wait() - end - end - end - return f -end - -function Base.copy(ctask::CTask) - task = ctask.task - if istaskfailed(task) - error("only runnable or finished tasks can be copied.") - end - - # memory copy - # newtask = ccall((:jl_clone_task, libtask_julia), Any, (Any,), task)::Task - newtask = ccall((:jl_clone_task_opaque, libtask_julia), - Any, (Any, Csize_t), task, TASK_OFFSETS[:END])::Task - internal_setfield(newtask, :exception, nothing) - internal_setfield(newtask, :backtrace, nothing) - internal_setfield(newtask, :tls, nothing) - internal_setfield(newtask, :result, nothing) - internal_setfield(newtask, :donenotify, nothing) - internal_setfield(newtask, :excstack, C_NULL) - internal_setfield(newtask, :ptls, C_NULL) - internal_setfield(newtask, :gcstack, C_NULL) - - if haskey(TASK_OFFSETS, :stkbuf) && haskey(TASK_OFFSETS, :bufsz) - old_stkbuf = internal_getfield(task, :stkbuf, Ptr) - if old_stkbuf != C_NULL - internal_setfield(task, :bufsz, Csize_t(0)) - else - internal_setfield(newtask, :stkbuf, C_NULL) - end - internal_setfield(newtask, :bufsz, Csize_t(0)) - end - memset(newtask, 0, :locks) - # memory copy done - - task.storage[:n_copies] = 1 + n_copies(task) - newtask.storage = copy(task.storage) - - # copy fields not accessible in task.c - newtask.code = task.code - setstate!(newtask, getstate(task)) - newtask.result = task.result - - copy_tarrays(task, newtask) - return CTask(newtask) -end - -function produce(v) - ct = _current_task() - ct.storage === nothing && (ct.storage = IdDict()) - - consumers = get!(ct.storage, :consumers, nothing) - local empty, task - while true - if consumers isa Task - task = consumers - ct.storage[:consumers] = nothing - empty = true - break - elseif consumers isa Condition && !isempty(consumers.waitq) - task = popfirst!(consumers.waitq) - empty = isempty(consumers.waitq) - break - end - - # Wait until there are more consumers. - wait() - - # Update consumers. - consumers = ct.storage[:consumers] - end - - # Internal check to make sure that it is possible to switch to the consumer. - @assert !istaskdone(task) && !istaskfailed(task) - - task.queue !== nothing && yield() - - if empty - # Switch to the consumer. - schedule(task, v) - wait() - - ct = _current_task() # When a task is copied, ct should be updated to new task ID. - while true - q = ct.storage[:consumers] - if isa(q,Task) - return q.result - elseif isa(q,Condition) && !isempty(q.waitq) - return q.waitq[1].result - end - - # Wait until there are more consumers. - wait() - end - else - schedule(task, v) - # make sure `task` runs before us. otherwise, the producer might - # finish before `t` runs again, causing it to see the producer - # as done, causing done(::Task, _) to miss the value `v`. - # see issue #7727 - yield() - return consumers.waitq[1].result - end -end - - -function consume(ctask::CTask, values...) - # Check if the producer is done. - producer = ctask.task - istaskdone(producer) && return wait(producer) - - # Obtain the current task and set its result. - ct = _current_task() - ct.result = length(values) == 1 ? values[1] : values - - # Obtain the consumers listening for the producer. - producer.storage === nothing && (producer.storage = IdDict()) - consumers = get!(producer.storage, :consumers, nothing) - - if consumers === nothing || (consumers isa Condition && isempty(consumers.waitq)) - # Set the current task as consumer if none exists. - producer.storage[:consumers] = ct - else - # Otherwise add the current task to the waiting queue. - if consumers isa Task - # If there is no queue currently but only a single task, replace it with a - # waiting queue with this task. - producer.storage[:consumers] = Condition() - push!(producer.storage[:consumers].waitq, consumers) - end - push!(producer.storage[:consumers].waitq, ct) - end - - if !istaskdone(producer) && !istaskfailed(producer) - # Switch to the producer. - schedule(producer) - yield() - - # Update the state if possible. - if producer.storage isa IdDict && haskey(producer.storage, :_libtask_state) - setstate!(producer, producer.storage[:_libtask_state]) - end - - # If the task failed, throw an exception. - istaskfailed(producer) && throw(CTaskException(producer)) - - # If the task is done return the result. - istaskdone(producer) && return producer.result - end - - wait() -end - -function getstate(task::Task) - @static if VERSION < v"1.6.0-DEV.618" - return task.state - else - return task._state - end -end - -function setstate!(task::Task, state) - @static if VERSION < v"1.6.0-DEV.618" - task.state = state - else - if state === :runnable - task._state = Base.task_state_runnable - elseif state === :done - task._state = Base.task_state_done - elseif state === :failed - task._state = Base.task_state_failed - else - task._state = state - end - end -end diff --git a/src/memlayout/darwin-x86_64-v1_3_1.jl b/src/memlayout/darwin-x86_64-v1_3_1.jl deleted file mode 100644 index f154b529..00000000 --- a/src/memlayout/darwin-x86_64-v1_3_1.jl +++ /dev/null @@ -1,28 +0,0 @@ -ALL_TASK_OFFSETS[("darwin-x86_64", v"1.3.1")] = Dict( - :END => 568, - :backtrace => 56, - :bufsz => 248, - :copy_stack => 256, - :ctx => 84, - :donenotify => 32, - :eh => 264, - :exception => 48, - :excstack => 280, - :gcstack => 272, - :locks => 304, - :logstate => 64, - :next => 0, - :prio => 298, - :queue => 8, - :result => 40, - :sizeof_ctx => 152, - :start => 72, - :state => 24, - :sticky => 80, - :stkbuf => 240, - :tid => 296, - :timing_stack => 560, - :tls => 16, - :tls_base_context => 6664, - :world_age => 288, -) diff --git a/src/memlayout/darwin-x86_64-v1_4_2.jl b/src/memlayout/darwin-x86_64-v1_4_2.jl deleted file mode 100644 index 82c858ed..00000000 --- a/src/memlayout/darwin-x86_64-v1_4_2.jl +++ /dev/null @@ -1,28 +0,0 @@ -ALL_TASK_OFFSETS[("darwin-x86_64", v"1.4.2")] = Dict( - :END => 568, - :backtrace => 56, - :bufsz => 248, - :copy_stack => 256, - :ctx => 84, - :donenotify => 32, - :eh => 264, - :exception => 48, - :excstack => 280, - :gcstack => 272, - :locks => 304, - :logstate => 64, - :next => 0, - :prio => 298, - :queue => 8, - :result => 40, - :sizeof_ctx => 152, - :start => 72, - :state => 24, - :sticky => 80, - :stkbuf => 240, - :tid => 296, - :timing_stack => 560, - :tls => 16, - :tls_base_context => 6664, - :world_age => 288, -) diff --git a/src/memlayout/darwin-x86_64-v1_5_3.jl b/src/memlayout/darwin-x86_64-v1_5_3.jl deleted file mode 100644 index 6d5bc272..00000000 --- a/src/memlayout/darwin-x86_64-v1_5_3.jl +++ /dev/null @@ -1,28 +0,0 @@ -ALL_TASK_OFFSETS[("darwin-x86_64", v"1.5.3")] = Dict( - :END => 568, - :backtrace => 56, - :bufsz => 248, - :copy_stack => 256, - :ctx => 84, - :donenotify => 32, - :eh => 264, - :exception => 48, - :excstack => 280, - :gcstack => 272, - :locks => 304, - :logstate => 64, - :next => 0, - :prio => 298, - :queue => 8, - :result => 40, - :sizeof_ctx => 152, - :start => 72, - :state => 24, - :sticky => 80, - :stkbuf => 240, - :tid => 296, - :timing_stack => 560, - :tls => 16, - :tls_base_context => 6672, - :world_age => 288, -) diff --git a/src/memlayout/darwin-x86_64-v1_5_4.jl b/src/memlayout/darwin-x86_64-v1_5_4.jl deleted file mode 100644 index ec6ddd4e..00000000 --- a/src/memlayout/darwin-x86_64-v1_5_4.jl +++ /dev/null @@ -1,27 +0,0 @@ -ALL_TASK_OFFSETS[("darwin-x86_64", v"1.5.4")] = Dict( - :END => 560, - :backtrace => 56, - :bufsz => 248, - :copy_stack => 256, - :ctx => 84, - :donenotify => 32, - :eh => 264, - :exception => 48, - :excstack => 280, - :gcstack => 272, - :locks => 296, - :logstate => 64, - :next => 0, - :prio => 290, - :queue => 8, - :result => 40, - :sizeof_ctx => 152, - :start => 72, - :state => 24, - :sticky => 80, - :stkbuf => 240, - :tid => 288, - :timing_stack => 552, - :tls => 16, - :tls_base_context => 6672, -) diff --git a/src/memlayout/darwin-x86_64-v1_6_1.jl b/src/memlayout/darwin-x86_64-v1_6_1.jl deleted file mode 100644 index cc4892eb..00000000 --- a/src/memlayout/darwin-x86_64-v1_6_1.jl +++ /dev/null @@ -1,24 +0,0 @@ -ALL_TASK_OFFSETS[("darwin-x86_64", v"1.6.1")] = Dict( - :END => 264, - :_isexception => 58, - :_state => 56, - :bufsz => 240, - :copy_stack => 248, - :ctx => 80, - :donenotify => 24, - :eh => 72, - :excstack => 64, - :gcstack => 256, - :logstate => 40, - :next => 0, - :prio => 62, - :queue => 8, - :result => 32, - :sizeof_ctx => 152, - :start => 48, - :sticky => 57, - :stkbuf => 232, - :tid => 60, - :tls => 16, - :tls_base_context => 6680, -) diff --git a/src/memlayout/linux-x86-v1_3_1.jl b/src/memlayout/linux-x86-v1_3_1.jl deleted file mode 100644 index 66461532..00000000 --- a/src/memlayout/linux-x86-v1_3_1.jl +++ /dev/null @@ -1,28 +0,0 @@ -ALL_TASK_OFFSETS[("linux-x86", v"1.3.1")] = Dict( - :END => 364, - :backtrace => 28, - :bufsz => 204, - :copy_stack => 208, - :ctx => 44, - :donenotify => 16, - :eh => 212, - :exception => 24, - :excstack => 220, - :gcstack => 216, - :locks => 232, - :logstate => 32, - :next => 0, - :prio => 230, - :queue => 4, - :result => 20, - :sizeof_ctx => 156, - :start => 36, - :state => 12, - :sticky => 40, - :stkbuf => 200, - :tid => 228, - :timing_stack => 360, - :tls => 8, - :tls_base_context => 3404, - :world_age => 224, -) diff --git a/src/memlayout/linux-x86-v1_4_2.jl b/src/memlayout/linux-x86-v1_4_2.jl deleted file mode 100644 index fa408f34..00000000 --- a/src/memlayout/linux-x86-v1_4_2.jl +++ /dev/null @@ -1,28 +0,0 @@ -ALL_TASK_OFFSETS[("linux-x86", v"1.4.2")] = Dict( - :END => 364, - :backtrace => 28, - :bufsz => 204, - :copy_stack => 208, - :ctx => 44, - :donenotify => 16, - :eh => 212, - :exception => 24, - :excstack => 220, - :gcstack => 216, - :locks => 232, - :logstate => 32, - :next => 0, - :prio => 230, - :queue => 4, - :result => 20, - :sizeof_ctx => 156, - :start => 36, - :state => 12, - :sticky => 40, - :stkbuf => 200, - :tid => 228, - :timing_stack => 360, - :tls => 8, - :tls_base_context => 3404, - :world_age => 224, -) diff --git a/src/memlayout/linux-x86-v1_5_3.jl b/src/memlayout/linux-x86-v1_5_3.jl deleted file mode 100644 index a6da71f9..00000000 --- a/src/memlayout/linux-x86-v1_5_3.jl +++ /dev/null @@ -1,28 +0,0 @@ -ALL_TASK_OFFSETS[("linux-x86", v"1.5.3")] = Dict( - :END => 364, - :backtrace => 28, - :bufsz => 204, - :copy_stack => 208, - :ctx => 44, - :donenotify => 16, - :eh => 212, - :exception => 24, - :excstack => 220, - :gcstack => 216, - :locks => 232, - :logstate => 32, - :next => 0, - :prio => 230, - :queue => 4, - :result => 20, - :sizeof_ctx => 156, - :start => 36, - :state => 12, - :sticky => 40, - :stkbuf => 200, - :tid => 228, - :timing_stack => 360, - :tls => 8, - :tls_base_context => 3408, - :world_age => 224, -) diff --git a/src/memlayout/linux-x86-v1_5_4.jl b/src/memlayout/linux-x86-v1_5_4.jl deleted file mode 100644 index 80d2bd9a..00000000 --- a/src/memlayout/linux-x86-v1_5_4.jl +++ /dev/null @@ -1,27 +0,0 @@ -ALL_TASK_OFFSETS[("linux-x86", v"1.5.4")] = Dict( - :END => 360, - :backtrace => 28, - :bufsz => 204, - :copy_stack => 208, - :ctx => 44, - :donenotify => 16, - :eh => 212, - :exception => 24, - :excstack => 220, - :gcstack => 216, - :locks => 228, - :logstate => 32, - :next => 0, - :prio => 226, - :queue => 4, - :result => 20, - :sizeof_ctx => 156, - :start => 36, - :state => 12, - :sticky => 40, - :stkbuf => 200, - :tid => 224, - :timing_stack => 356, - :tls => 8, - :tls_base_context => 3408, -) diff --git a/src/memlayout/linux-x86-v1_6_1.jl b/src/memlayout/linux-x86-v1_6_1.jl deleted file mode 100644 index 8d04faee..00000000 --- a/src/memlayout/linux-x86-v1_6_1.jl +++ /dev/null @@ -1,24 +0,0 @@ -ALL_TASK_OFFSETS[("linux-x86", v"1.6.1")] = Dict( - :END => 216, - :_isexception => 30, - :_state => 28, - :bufsz => 204, - :copy_stack => 208, - :ctx => 44, - :donenotify => 12, - :eh => 40, - :excstack => 36, - :gcstack => 212, - :logstate => 20, - :next => 0, - :prio => 34, - :queue => 4, - :result => 16, - :sizeof_ctx => 156, - :start => 24, - :sticky => 29, - :stkbuf => 200, - :tid => 32, - :tls => 8, - :tls_base_context => 3412, -) diff --git a/src/memlayout/linux-x86-v1_8_0.jl b/src/memlayout/linux-x86-v1_8_0.jl deleted file mode 100644 index e681843b..00000000 --- a/src/memlayout/linux-x86-v1_8_0.jl +++ /dev/null @@ -1,30 +0,0 @@ -ALL_TASK_OFFSETS[("linux-x86", v"1.8.0")] = Dict( - :END => 256, - :_isexception => 30, - :_state => 28, - :bufsz => 248, - :copy_stack => 252, - :ctx => 88, - :donenotify => 12, - :eh => 84, - :excstack => 80, - :gcstack => 64, - :logstate => 20, - :next => 0, - :prio => 78, - :ptls => 72, - :queue => 4, - :result => 16, - :rngState0 => 32, - :rngState1 => 40, - :rngState2 => 48, - :rngState3 => 56, - :sizeof_ctx => 156, - :start => 24, - :sticky => 29, - :stkbuf => 244, - :tid => 76, - :tls => 8, - :tls_base_context => 3416, - :world_age => 68, -) diff --git a/src/memlayout/linux-x86_64-v1_3_1.jl b/src/memlayout/linux-x86_64-v1_3_1.jl deleted file mode 100644 index cca19241..00000000 --- a/src/memlayout/linux-x86_64-v1_3_1.jl +++ /dev/null @@ -1,28 +0,0 @@ -ALL_TASK_OFFSETS[("linux-x86_64", v"1.3.1")] = Dict( - :END => 616, - :backtrace => 56, - :bufsz => 296, - :copy_stack => 304, - :ctx => 88, - :donenotify => 32, - :eh => 312, - :exception => 48, - :excstack => 328, - :gcstack => 320, - :locks => 352, - :logstate => 64, - :next => 0, - :prio => 346, - :queue => 8, - :result => 40, - :sizeof_ctx => 200, - :start => 72, - :state => 24, - :sticky => 80, - :stkbuf => 288, - :tid => 344, - :timing_stack => 608, - :tls => 16, - :tls_base_context => 6640, - :world_age => 336, -) diff --git a/src/memlayout/linux-x86_64-v1_4_2.jl b/src/memlayout/linux-x86_64-v1_4_2.jl deleted file mode 100644 index 6a1cc083..00000000 --- a/src/memlayout/linux-x86_64-v1_4_2.jl +++ /dev/null @@ -1,28 +0,0 @@ -ALL_TASK_OFFSETS[("linux-x86_64", v"1.4.2")] = Dict( - :END => 616, - :backtrace => 56, - :bufsz => 296, - :copy_stack => 304, - :ctx => 88, - :donenotify => 32, - :eh => 312, - :exception => 48, - :excstack => 328, - :gcstack => 320, - :locks => 352, - :logstate => 64, - :next => 0, - :prio => 346, - :queue => 8, - :result => 40, - :sizeof_ctx => 200, - :start => 72, - :state => 24, - :sticky => 80, - :stkbuf => 288, - :tid => 344, - :timing_stack => 608, - :tls => 16, - :tls_base_context => 6640, - :world_age => 336, -) diff --git a/src/memlayout/linux-x86_64-v1_5_3.jl b/src/memlayout/linux-x86_64-v1_5_3.jl deleted file mode 100644 index 2e92d6d7..00000000 --- a/src/memlayout/linux-x86_64-v1_5_3.jl +++ /dev/null @@ -1,28 +0,0 @@ -ALL_TASK_OFFSETS[("linux-x86_64", v"1.5.3")] = Dict( - :END => 616, - :backtrace => 56, - :bufsz => 296, - :copy_stack => 304, - :ctx => 88, - :donenotify => 32, - :eh => 312, - :exception => 48, - :excstack => 328, - :gcstack => 320, - :locks => 352, - :logstate => 64, - :next => 0, - :prio => 346, - :queue => 8, - :result => 40, - :sizeof_ctx => 200, - :start => 72, - :state => 24, - :sticky => 80, - :stkbuf => 288, - :tid => 344, - :timing_stack => 608, - :tls => 16, - :tls_base_context => 6648, - :world_age => 336, -) diff --git a/src/memlayout/linux-x86_64-v1_5_4.jl b/src/memlayout/linux-x86_64-v1_5_4.jl deleted file mode 100644 index 505955d0..00000000 --- a/src/memlayout/linux-x86_64-v1_5_4.jl +++ /dev/null @@ -1,27 +0,0 @@ -ALL_TASK_OFFSETS[("linux-x86_64", v"1.5.4")] = Dict( - :END => 608, - :backtrace => 56, - :bufsz => 296, - :copy_stack => 304, - :ctx => 88, - :donenotify => 32, - :eh => 312, - :exception => 48, - :excstack => 328, - :gcstack => 320, - :locks => 344, - :logstate => 64, - :next => 0, - :prio => 338, - :queue => 8, - :result => 40, - :sizeof_ctx => 200, - :start => 72, - :state => 24, - :sticky => 80, - :stkbuf => 288, - :tid => 336, - :timing_stack => 600, - :tls => 16, - :tls_base_context => 6648, -) diff --git a/src/memlayout/linux-x86_64-v1_6_1.jl b/src/memlayout/linux-x86_64-v1_6_1.jl deleted file mode 100644 index adbe2774..00000000 --- a/src/memlayout/linux-x86_64-v1_6_1.jl +++ /dev/null @@ -1,24 +0,0 @@ -ALL_TASK_OFFSETS[("linux-x86_64", v"1.6.1")] = Dict( - :END => 312, - :_isexception => 58, - :_state => 56, - :bufsz => 288, - :copy_stack => 296, - :ctx => 80, - :donenotify => 24, - :eh => 72, - :excstack => 64, - :gcstack => 304, - :logstate => 40, - :next => 0, - :prio => 62, - :queue => 8, - :result => 32, - :sizeof_ctx => 200, - :start => 48, - :sticky => 57, - :stkbuf => 280, - :tid => 60, - :tls => 16, - :tls_base_context => 6656, -) diff --git a/src/memlayout/linux-x86_64-v1_8_0.jl b/src/memlayout/linux-x86_64-v1_8_0.jl deleted file mode 100644 index 6b2a97ce..00000000 --- a/src/memlayout/linux-x86_64-v1_8_0.jl +++ /dev/null @@ -1,30 +0,0 @@ -ALL_TASK_OFFSETS[("linux-x86_64", v"1.8.0")] = Dict( - :END => 368, - :_isexception => 58, - :_state => 56, - :bufsz => 352, - :copy_stack => 360, - :ctx => 144, - :donenotify => 24, - :eh => 136, - :excstack => 128, - :gcstack => 96, - :logstate => 40, - :next => 0, - :prio => 122, - :ptls => 112, - :queue => 8, - :result => 32, - :rngState0 => 64, - :rngState1 => 72, - :rngState2 => 80, - :rngState3 => 88, - :sizeof_ctx => 200, - :start => 48, - :sticky => 57, - :stkbuf => 344, - :tid => 120, - :tls => 16, - :tls_base_context => 6656, - :world_age => 104, -) diff --git a/src/memlayout/main.jl b/src/memlayout/main.jl deleted file mode 100644 index 169cc93a..00000000 --- a/src/memlayout/main.jl +++ /dev/null @@ -1,106 +0,0 @@ -const ALL_TASK_OFFSETS = - Dict{Tuple{String, VersionNumber}, Dict{Symbol, Int}}() - - -include("linux-x86-v1_3_1.jl") -include("linux-x86-v1_4_2.jl") -include("linux-x86-v1_5_3.jl") -include("linux-x86-v1_5_4.jl") -include("linux-x86-v1_6_1.jl") -include("linux-x86-v1_8_0.jl") - -include("linux-x86_64-v1_3_1.jl") -include("linux-x86_64-v1_4_2.jl") -include("linux-x86_64-v1_5_3.jl") -include("linux-x86_64-v1_5_4.jl") -include("linux-x86_64-v1_6_1.jl") -include("linux-x86_64-v1_8_0.jl") - -include("darwin-x86_64-v1_3_1.jl") -include("darwin-x86_64-v1_4_2.jl") -include("darwin-x86_64-v1_5_3.jl") -include("darwin-x86_64-v1_5_4.jl") -include("darwin-x86_64-v1_6_1.jl") - -include("windows-x86-v1_3_1.jl") -include("windows-x86-v1_4_2.jl") -include("windows-x86-v1_5_3.jl") -include("windows-x86-v1_5_4.jl") -include("windows-x86-v1_6_1.jl") - -include("windows-x86_64-v1_3_1.jl") -include("windows-x86_64-v1_4_2.jl") -include("windows-x86_64-v1_5_3.jl") -include("windows-x86_64-v1_5_4.jl") -include("windows-x86_64-v1_6_1.jl") - -const ARCH = @static if string(Sys.ARCH)[1] == 'i' - "x86" -else - string(Sys.ARCH) -end - -const PLATFORM = @static if Sys.islinux() - "linux-" * ARCH -elseif Sys.iswindows() - "windows-" * ARCH -elseif Sys.isapple() - "darwin-" * ARCH -end - -function find_offsets() - candi = [v for (p, v) in keys(ALL_TASK_OFFSETS) if p == PLATFORM] - sort!(candi) - candi = [v for v in candi if v <= VERSION] - isempty(candi) && error("No suitable offsets") - ALL_TASK_OFFSETS[(PLATFORM, candi[end])] -end - -const TASK_OFFSETS = find_offsets() - -## utilities - -# getter -function internal_getfield(task, field, ::Type{Ptr}) - ccall((:jl_getfield_ptr, libtask_julia), - Ptr{Cvoid}, (Any, Csize_t), - task, TASK_OFFSETS[field]) -end - -function internal_getfield(task, field, ::Type{Int32}) - ccall((:jl_getfield_int32_t, libtask_julia), - Int32, (Any, Csize_t), task, TASK_OFFSETS[field]) -end - -# setter -function internal_setfield(task, field, ::Nothing) - haskey(TASK_OFFSETS, field) && ccall( - (:jl_setfield_nothing, libtask_julia), - Any, (Any, Csize_t), task, TASK_OFFSETS[field]) -end - -function internal_setfield(task, field, p::Ptr{Nothing}) - p == C_NULL || error("this function is only for setting NULL value") - haskey(TASK_OFFSETS, field) && ccall( - (:jl_setfield_null, libtask_julia), - Any, (Any, Csize_t), task, TASK_OFFSETS[field]) -end - -function internal_setfield(task, field, val::Csize_t) - ccall((:jl_setfield_size_t, libtask_julia), - Any, (Any, Csize_t, Csize_t), - task, TASK_OFFSETS[field], val) -end - -function internal_setfield(task, field, val::Int32) - ccall((:jl_setfield_int32_t, libtask_julia), - Any, (Any, Csize_t, Int32), - task, TASK_OFFSETS[field], val) -end - -function memset(task, val, offset_field; end_field=:END) - haskey(TASK_OFFSETS, offset_field) && ccall( - (:jl_memset, libtask_julia), - Cvoid, (Any, Csize_t, Csize_t, Int), - task, TASK_OFFSETS[offset_field], TASK_OFFSETS[end_field], val) -end diff --git a/src/memlayout/windows-x86-v1_3_1.jl b/src/memlayout/windows-x86-v1_3_1.jl deleted file mode 100644 index 6fe2d316..00000000 --- a/src/memlayout/windows-x86-v1_3_1.jl +++ /dev/null @@ -1,28 +0,0 @@ -ALL_TASK_OFFSETS[("windows-x86", v"1.3.1")] = Dict( - :END => 280, - :backtrace => 28, - :bufsz => 120, - :copy_stack => 124, - :ctx => 44, - :donenotify => 16, - :eh => 128, - :exception => 24, - :excstack => 136, - :gcstack => 132, - :locks => 148, - :logstate => 32, - :next => 0, - :prio => 146, - :queue => 4, - :result => 20, - :sizeof_ctx => 72, - :start => 36, - :state => 12, - :sticky => 40, - :stkbuf => 116, - :tid => 144, - :timing_stack => 276, - :tls => 8, - :tls_base_context => 3352, - :world_age => 140, -) diff --git a/src/memlayout/windows-x86-v1_4_2.jl b/src/memlayout/windows-x86-v1_4_2.jl deleted file mode 100644 index 1f502163..00000000 --- a/src/memlayout/windows-x86-v1_4_2.jl +++ /dev/null @@ -1,28 +0,0 @@ -ALL_TASK_OFFSETS[("windows-x86", v"1.4.2")] = Dict( - :END => 280, - :backtrace => 28, - :bufsz => 120, - :copy_stack => 124, - :ctx => 44, - :donenotify => 16, - :eh => 128, - :exception => 24, - :excstack => 136, - :gcstack => 132, - :locks => 148, - :logstate => 32, - :next => 0, - :prio => 146, - :queue => 4, - :result => 20, - :sizeof_ctx => 72, - :start => 36, - :state => 12, - :sticky => 40, - :stkbuf => 116, - :tid => 144, - :timing_stack => 276, - :tls => 8, - :tls_base_context => 3352, - :world_age => 140, -) diff --git a/src/memlayout/windows-x86-v1_5_3.jl b/src/memlayout/windows-x86-v1_5_3.jl deleted file mode 100644 index 7c73b7a8..00000000 --- a/src/memlayout/windows-x86-v1_5_3.jl +++ /dev/null @@ -1,28 +0,0 @@ -ALL_TASK_OFFSETS[("windows-x86", v"1.5.3")] = Dict( - :END => 280, - :backtrace => 28, - :bufsz => 120, - :copy_stack => 124, - :ctx => 44, - :donenotify => 16, - :eh => 128, - :exception => 24, - :excstack => 136, - :gcstack => 132, - :locks => 148, - :logstate => 32, - :next => 0, - :prio => 146, - :queue => 4, - :result => 20, - :sizeof_ctx => 72, - :start => 36, - :state => 12, - :sticky => 40, - :stkbuf => 116, - :tid => 144, - :timing_stack => 276, - :tls => 8, - :tls_base_context => 3356, - :world_age => 140, -) diff --git a/src/memlayout/windows-x86-v1_5_4.jl b/src/memlayout/windows-x86-v1_5_4.jl deleted file mode 100644 index 53f9c6dc..00000000 --- a/src/memlayout/windows-x86-v1_5_4.jl +++ /dev/null @@ -1,27 +0,0 @@ -ALL_TASK_OFFSETS[("windows-x86", v"1.5.4")] = Dict( - :END => 276, - :backtrace => 28, - :bufsz => 120, - :copy_stack => 124, - :ctx => 44, - :donenotify => 16, - :eh => 128, - :exception => 24, - :excstack => 136, - :gcstack => 132, - :locks => 144, - :logstate => 32, - :next => 0, - :prio => 142, - :queue => 4, - :result => 20, - :sizeof_ctx => 72, - :start => 36, - :state => 12, - :sticky => 40, - :stkbuf => 116, - :tid => 140, - :timing_stack => 272, - :tls => 8, - :tls_base_context => 3356, -) diff --git a/src/memlayout/windows-x86-v1_6_1.jl b/src/memlayout/windows-x86-v1_6_1.jl deleted file mode 100644 index 2b1f3750..00000000 --- a/src/memlayout/windows-x86-v1_6_1.jl +++ /dev/null @@ -1,24 +0,0 @@ -ALL_TASK_OFFSETS[("windows-x86", v"1.6.1")] = Dict( - :END => 132, - :_isexception => 30, - :_state => 28, - :bufsz => 120, - :copy_stack => 124, - :ctx => 44, - :donenotify => 12, - :eh => 40, - :excstack => 36, - :gcstack => 128, - :logstate => 20, - :next => 0, - :prio => 34, - :queue => 4, - :result => 16, - :sizeof_ctx => 72, - :start => 24, - :sticky => 29, - :stkbuf => 116, - :tid => 32, - :tls => 8, - :tls_base_context => 3360, -) diff --git a/src/memlayout/windows-x86_64-v1_3_1.jl b/src/memlayout/windows-x86_64-v1_3_1.jl deleted file mode 100644 index 1669f3f8..00000000 --- a/src/memlayout/windows-x86_64-v1_3_1.jl +++ /dev/null @@ -1,28 +0,0 @@ -ALL_TASK_OFFSETS[("windows-x86_64", v"1.3.1")] = Dict( - :END => 704, - :backtrace => 56, - :bufsz => 376, - :copy_stack => 384, - :ctx => 96, - :donenotify => 32, - :eh => 392, - :exception => 48, - :excstack => 408, - :gcstack => 400, - :locks => 432, - :logstate => 64, - :next => 0, - :prio => 426, - :queue => 8, - :result => 40, - :sizeof_ctx => 272, - :start => 72, - :state => 24, - :sticky => 80, - :stkbuf => 368, - :tid => 424, - :timing_stack => 688, - :tls => 16, - :tls_base_context => 6608, - :world_age => 416, -) diff --git a/src/memlayout/windows-x86_64-v1_4_2.jl b/src/memlayout/windows-x86_64-v1_4_2.jl deleted file mode 100644 index 504caa71..00000000 --- a/src/memlayout/windows-x86_64-v1_4_2.jl +++ /dev/null @@ -1,28 +0,0 @@ -ALL_TASK_OFFSETS[("windows-x86_64", v"1.4.2")] = Dict( - :END => 704, - :backtrace => 56, - :bufsz => 376, - :copy_stack => 384, - :ctx => 96, - :donenotify => 32, - :eh => 392, - :exception => 48, - :excstack => 408, - :gcstack => 400, - :locks => 432, - :logstate => 64, - :next => 0, - :prio => 426, - :queue => 8, - :result => 40, - :sizeof_ctx => 272, - :start => 72, - :state => 24, - :sticky => 80, - :stkbuf => 368, - :tid => 424, - :timing_stack => 688, - :tls => 16, - :tls_base_context => 6608, - :world_age => 416, -) diff --git a/src/memlayout/windows-x86_64-v1_5_3.jl b/src/memlayout/windows-x86_64-v1_5_3.jl deleted file mode 100644 index 07d4ebae..00000000 --- a/src/memlayout/windows-x86_64-v1_5_3.jl +++ /dev/null @@ -1,28 +0,0 @@ -ALL_TASK_OFFSETS[("windows-x86_64", v"1.5.3")] = Dict( - :END => 704, - :backtrace => 56, - :bufsz => 376, - :copy_stack => 384, - :ctx => 96, - :donenotify => 32, - :eh => 392, - :exception => 48, - :excstack => 408, - :gcstack => 400, - :locks => 432, - :logstate => 64, - :next => 0, - :prio => 426, - :queue => 8, - :result => 40, - :sizeof_ctx => 272, - :start => 72, - :state => 24, - :sticky => 80, - :stkbuf => 368, - :tid => 424, - :timing_stack => 688, - :tls => 16, - :tls_base_context => 6608, - :world_age => 416, -) diff --git a/src/memlayout/windows-x86_64-v1_5_4.jl b/src/memlayout/windows-x86_64-v1_5_4.jl deleted file mode 100644 index 446c1396..00000000 --- a/src/memlayout/windows-x86_64-v1_5_4.jl +++ /dev/null @@ -1,27 +0,0 @@ -ALL_TASK_OFFSETS[("windows-x86_64", v"1.5.4")] = Dict( - :END => 688, - :backtrace => 56, - :bufsz => 376, - :copy_stack => 384, - :ctx => 96, - :donenotify => 32, - :eh => 392, - :exception => 48, - :excstack => 408, - :gcstack => 400, - :locks => 424, - :logstate => 64, - :next => 0, - :prio => 418, - :queue => 8, - :result => 40, - :sizeof_ctx => 272, - :start => 72, - :state => 24, - :sticky => 80, - :stkbuf => 368, - :tid => 416, - :timing_stack => 680, - :tls => 16, - :tls_base_context => 6608, -) diff --git a/src/memlayout/windows-x86_64-v1_6_1.jl b/src/memlayout/windows-x86_64-v1_6_1.jl deleted file mode 100644 index 6640becd..00000000 --- a/src/memlayout/windows-x86_64-v1_6_1.jl +++ /dev/null @@ -1,24 +0,0 @@ -ALL_TASK_OFFSETS[("windows-x86_64", v"1.6.1")] = Dict( - :END => 384, - :_isexception => 58, - :_state => 56, - :bufsz => 360, - :copy_stack => 368, - :ctx => 80, - :donenotify => 24, - :eh => 72, - :excstack => 64, - :gcstack => 376, - :logstate => 40, - :next => 0, - :prio => 62, - :queue => 8, - :result => 32, - :sizeof_ctx => 272, - :start => 48, - :sticky => 57, - :stkbuf => 352, - :tid => 60, - :tls => 16, - :tls_base_context => 6624, -) diff --git a/src/tapedfunction.jl b/src/tapedfunction.jl new file mode 100644 index 00000000..009f3f4a --- /dev/null +++ b/src/tapedfunction.jl @@ -0,0 +1,221 @@ +mutable struct Instruction{F} + fun::F + input::Tuple + output + tape +end + + +mutable struct Tape + tape::Vector{Instruction} + owner +end + +Tape() = Tape(Vector{Instruction}(), nothing) +Tape(owner) = Tape(Vector{Instruction}(), owner) +MacroTools.@forward Tape.tape Base.iterate, Base.length +MacroTools.@forward Tape.tape Base.push!, Base.getindex, Base.lastindex +const NULL_TAPE = Tape() + +mutable struct Box{T} + val::T +end + +val(x) = x +val(x::Box) = x.val +box(x) = Box(x) +any_box(x) = Box{Any}(x) + +gettape(x) = nothing +gettape(x::Instruction) = x.tape +function gettape(x::Tuple) + for i in x + gettape(i) != nothing && return gettape(i) + end +end +result(t::Tape) = isempty(t) ? nothing : val(t[end].output) + +function Base.show(io::IO, box::Box) + println(io, "Box($(box.val))") +end + +function Base.show(io::IO, instruction::Instruction) + fun = instruction.fun + tape = instruction.tape + println(io, "Instruction($(fun)), tape=$(objectid(tape)))") +end + +function Base.show(io::IO, tp::Tape) + buf = IOBuffer() + print(buf, "$(length(tp))-element Tape") + isempty(tp) || println(buf, ":") + i = 1 + for instruction in tp + print(buf, "\t$i => ") + show(buf, instruction) + i += 1 + end + print(io, String(take!(buf))) +end + +function (instr::Instruction{F})() where F + output = instr.fun(map(val, instr.input)...) + instr.output.val = output +end + +function run(tape::Tape, args...) + input = map(box, args) + tape[1].input = input + for instruction in tape + instruction() + end +end + +function run_and_record!(tape::Tape, f, args...) + f = val(f) # f maybe a Boxed closure + output = try + box(f(map(val, args)...)) + catch + any_box(nothing) + end + ins = Instruction(f, args, output, tape) + push!(tape, ins) + return output +end + +function dry_record!(tape::Tape, f, args...) + # We don't know the type of box.val now, so we use Box{Any} + output = any_box(nothing) + ins = Instruction(f, args, output, tape) + push!(tape, ins) + return output +end + +function unbox_condition(ir) + for blk in IRTools.blocks(ir) + vars = keys(blk) + for br in IRTools.branches(blk) + IRTools.isconditional(br) || continue + cond = br.condition + prev_cond = IRTools.insert!(ir, cond, ir[cond]) + ir[cond] = IRTools.xcall(@__MODULE__, :val, prev_cond) + end + end +end + +box_args() = nothing +box_args(x) = x +box_args(args...) = args + +function _replace_args(args, pairs::Dict) + map(args) do x + haskey(pairs, x) ? pairs[x] : x + end +end + +function intercept(ir; recorder=:run_and_record!) + ir == nothing && return + tape = pushfirst!(ir, IRTools.xcall(@__MODULE__, :Tape)) + + # box the args + first_blk = IRTools.blocks(ir)[1] + args = IRTools.arguments(first_blk) + arity = length(args) - 1 + arg_pairs= Dict() + + args_var = args[1] + if arity == 0 + args_var = IRTools.insertafter!(ir, tape, IRTools.xcall(@__MODULE__, :box_args)) + elseif arity == 1 + args_var = IRTools.insertafter!(ir, tape, IRTools.xcall(@__MODULE__, :box_args, args[2])) + arg_pairs = Dict(args[2] => args_var) + else # arity > 1 + args_var = IRTools.insertafter!(ir, tape, IRTools.xcall(@__MODULE__, :box_args, args[2:end]...)) + args_new, last_pos = [], args_var + + iter_state = [] + + for i in 1:arity + last_pos = IRTools.insertafter!(ir, last_pos, IRTools.xcall(Base, :indexed_iterate, args_var, i, iter_state...)) + args_iter = last_pos + last_pos = IRTools.insertafter!(ir, last_pos, IRTools.xcall(Core, :getfield, args_iter, 1)) + push!(args_new, last_pos) + if i != arity + last_pos = IRTools.insertafter!(ir, last_pos, IRTools.xcall(Core, :getfield, args_iter, 2)) + iter_state = [last_pos] + end + end + arg_pairs = Dict(zip(args[2:end], args_new)) + end + + # here we assumed the ir only has a return statement at its last block, + # and we make sure the return value is from a function call (to `identity`) + last_blk = IRTools.blocks(ir)[end] + retv = IRTools.returnvalue(last_blk) + IRTools.return!(last_blk, IRTools.xcall(Base, :identity, retv)) + + for (x, st) in ir + x == tape && continue + Meta.isexpr(st.expr, :call) || continue + new_args = (x == args_var) ? st.expr.args : _replace_args(st.expr.args, arg_pairs) + ir[x] = IRTools.xcall(@__MODULE__, recorder, tape, new_args...) + end + # the real return value will be in the last instruction on the tape + IRTools.return!(ir, tape) + unbox_condition(ir) + return ir +end + +mutable struct TapedFunction + func # ::Function # maybe a callable obejct + arity::Int + ir::Union{Nothing, IRTools.IR} + tape::Tape + owner + function TapedFunction(f; arity::Int=-1) + new(f, arity, nothing, NULL_TAPE, nothing) + end +end + +function (tf::TapedFunction)(args...) + if isempty(tf.tape) + ir = IRTools.@code_ir tf.func(args...) + ir = intercept(ir; recorder=:run_and_record!) + tape = IRTools.evalir(ir, tf.func, args...) + tf.ir = ir + tf.tape = tape + tape.owner = tf + return result(tape) + end + # TODO: use cache + run(tf.tape, args...) + return result(tf.tape) +end + +function dry_run(tf::TapedFunction) + isempty(tf.tape) || (return tf) + @assert tf.arity >= 0 "TapedFunction need a fixed arity to dry run." + args = fill(nothing, tf.arity) + ir = IRTools.@code_ir tf.func(args...) + ir = intercept(ir; recorder=:dry_record!) + tape = IRTools.evalir(ir, tf.func, args...) + tf.ir = ir + tf.tape = tape + tape.owner = tf + return tf +end + +function Base.show(io::IO, tf::TapedFunction) + buf = IOBuffer() + println(buf, "TapedFunction:") + println(buf, "* .func => $(tf.func)") + println(buf, "* .ir =>") + println(buf, "------------------") + println(buf, tf.ir) + println(buf, "------------------") + println(buf, "* .tape =>") + println(buf, "------------------") + println(buf, tf.tape) + println(buf, "------------------") + print(io, String(take!(buf))) +end diff --git a/src/tapedtask.jl b/src/tapedtask.jl new file mode 100644 index 00000000..5864c607 --- /dev/null +++ b/src/tapedtask.jl @@ -0,0 +1,199 @@ +struct TapedTaskException + exc +end + +struct TapedTask + task::Task + tf::TapedFunction + counter::Ref{Int} + produce_ch::Channel{Any} + consume_ch::Channel{Int} +end + +function TapedTask(tf::TapedFunction, args...) + tf.owner != nothing && error("TapedFunction is owned to another task.") + # dry_run(tf) + isempty(tf.tape) && tf(args...) + counter = Ref{Int}(1) + produce_ch = Channel() + consume_ch = Channel{Int}() + task = @task try + step_in(tf, counter, args) + catch e + put!(produce_ch, TapedTaskException(e)) + # @error "TapedTask Error: " exception=(e, catch_backtrace()) + rethrow() + finally + while !isempty(produce_ch) + yield() + end + close(produce_ch) + close(consume_ch) + end + t = TapedTask(task, tf, counter, produce_ch, consume_ch) + # task.storage === nothing && (task.storage = IdDict()) + # task.storage[:tapedtask] = t + tf.owner = t + return t +end + +TapedTask(f, args...) = TapedTask(TapedFunction(f, arity=length(args)), args...) +TapedTask(t::TapedTask, args...) = TapedTask(func(t), args...) +func(t::TapedTask) = t.tf.func + +function step_in(tf::TapedFunction, counter::Ref{Int}, args) + len = length(tf.tape) + if(counter[] <= 1) + input = map(box, args) + tf.tape[1].input = input + end + while counter[] <= len + tf.tape[counter[]]() + counter[] += 1 + end +end + +# A way (the old way) to impl `produce`, which does NOT +# support `produce` in a nested call +function internal_produce(instr::Instruction, val) + tape = gettape(instr) + tf = tape.owner + ttask = tf.owner + put!(ttask.produce_ch, val) + take!(ttask.consume_ch) # wait for next consumer +end + +function produce(val) + error("Libtask.produce can only be directly called in a task!") +end + +function (instr::Instruction{typeof(produce)})() + args = val(instr.input[1]) + internal_produce(instr, args) +end + +#= +# Another way to support `produce` in nested call. This way has its caveat: +# `produce` may deeply hide in an instruction, but not be an instruction +# itself, and when we copy a task, the newly copied task will resume from +# the instruction after the one which contains this `produce` call. If the +# call to `produce` is not the last expression in the instuction, that +# instruction will not be whole executed in the copied task. +@inline function is_in_tapedtask() + ct = current_task() + ct.storage === nothing && return false + haskey(ct.storage, :tapedtask) || return false + # check if we are recording a tape + ct.storage[:tapedtask].tf.tape === NULL_TAPE && return false + return true +end + +function produce(val) + is_in_tapedtask() || return nothing + ttask = current_task().storage[:tapedtask] + put!(ttask.produce_ch, val) + take!(ttask.consume_ch) # wait for next consumer + return nothing +end +=# + +function consume(ttask::TapedTask) + if istaskstarted(ttask.task) + # tell producer that a consumer is coming + put!(ttask.consume_ch, 0) + else + schedule(ttask.task) + end + + val = try + take!(ttask.produce_ch) + catch e + isa(e, InvalidStateException) || rethrow() + istaskfailed(ttask.task) && throw(ttask.task.exception) + # TODO: we return nothing to indicate the end of a task, + # remove this when AdvancedPS is udpated. + istaskdone(ttask.task) && return nothing + end + + # yield to let the task resume, this is necessary when there's + # an exception is thrown in the task, it gives the task the chance + # to rethow the exception and set its proper status: + yield() + isa(val, TapedTaskException) && throw(val.exc) + return val +end + +# Iteration interface. +function Base.iterate(t::TapedTask, state=nothing) + try + consume(t), nothing + catch ex + !isa(ex, InvalidStateException) && rethrow + nothing + end +end +Base.IteratorSize(::Type{TapedTask}) = Base.SizeUnknown() +Base.IteratorEltype(::Type{TapedTask}) = Base.EltypeUnknown() + + +# copy the task + +""" + tape_copy(x) + +Function `tape_copy` is used to copy data while copying a TapedTask, the +default behavior is: 1. for `Array` and `Dict`, we do `deepcopy`; 2. for +other data types, we do not copy and share the data between tasks, i.e., +`tape_copy(x) = x`. If one wants some kinds of data to be copied, or +deeply copied, one can add a method to this function. +""" +function tape_copy end +tape_copy(x) = x +# tape_copy(x::Array) = deepcopy(x) +# tape_copy(x::Dict) = deepcopy(x) + +function copy_box(old_box::Box{T}, roster::Dict{UInt64, Any}) where T + oid = objectid(old_box) + haskey(roster, oid) && (return roster[oid]) + + # We don't know the type of box.val now, so we use Box{Any} + new_box = Box{T}(tape_copy(old_box.val)) + roster[oid] = new_box + return new_box +end +copy_box(o, roster::Dict{UInt64, Any}) = o + +function Base.copy(t::Tape) + old_data = t.tape + new_data = Vector{Instruction}() + new_tape = Tape(new_data, t.owner) + + roster = Dict{UInt64, Any}() + for x in old_data + input = map(x.input) do ob + copy_box(ob, roster) + end + output = copy_box(x.output, roster) + new_ins = Instruction(x.fun, input, output, new_tape) + push!(new_data, new_ins) + end + + return new_tape +end + +function Base.copy(tf::TapedFunction) + new_tf = TapedFunction(tf.func; arity=tf.arity) + new_tf.ir = tf.ir + new_tape = copy(tf.tape) + new_tape.owner = new_tf + new_tf.tape = new_tape + return new_tf +end + +function Base.copy(t::TapedTask) + # t.counter[] <= 1 && error("Can't copy a TapedTask which is not running.") + tf = copy(t.tf) + new_t = TapedTask(tf) + new_t.counter[] = t.counter[] + 1 + return new_t +end diff --git a/src/tarray.jl b/src/tarray.jl index c7c4451b..507f59c5 100644 --- a/src/tarray.jl +++ b/src/tarray.jl @@ -5,17 +5,11 @@ """ TArray{T}(dims, ...) -Implementation of data structures that automatically perform copy-on-write after task copying. - -For each `TArray` object, we have a task-local array object, i.e. if -you access (read or write) the same `TArray` object in different -tasks, you may be dealing with different array objects each belonging -to a different task. These task-specific array objects, however, -share the same parent `TArray` object. +Implementation of data structures that automatically perform deepcopy during task copying. More specifically, we store the underlying arrays in the field -`TArray.data`, a dictionary whose keys are tasks, instead of storing -them in the task local storage, for performance considerations. +`TArray.data`, and this filed will be deepcopied while we are copying +a task who manipulates it. Usage: @@ -32,14 +26,7 @@ Array(ta) # convert to 4-element Array{Int64,1}: [1, 2, 3, 4] ``` """ mutable struct TArray{T, N, A <: AbstractArray{T, N}} <: AbstractArray{T, N} - orig_task :: Task - data::Dict{Task, Tuple{Int, A}} - function TArray{T, N, A}() where {T, N, A <: AbstractArray{T, N}} - d = Dict{Task, Tuple{Int, A}}() - res = new(current_task(), d) - register_to_keeper(res) - return res - end + data::A end TArray{T}(d::Integer...) where T = TArray(T, d) @@ -48,58 +35,17 @@ TArray{T}(::UndefInitializer, dim::NTuple{N,Int}) where {T,N} = TArray(T, dim) TArray{T,N}(d::Vararg{<:Integer,N}) where {T,N} = TArray(T, d) TArray{T,N}(::UndefInitializer, d::Vararg{<:Integer,N}) where {T,N} = TArray{T,N}(d) TArray{T,N}(dim::NTuple{N,Int}) where {T,N} = TArray(T, dim) +TArray(T::Type, dim) = TArray(Array{T}(undef, dim)) -function TArray(T::Type, dim) - N_dim = length(dim) - res = TArray{T, N_dim, Array{T, N_dim}}() - n = n_copies() - d = Array{T}(undef, dim) - _set_local_storage(res, n, d) - res -end - -TArray(x::AbstractArray) = convert(TArray, x) - -# TArray House-Keeper -const TArrayKeeper = Vector{WeakRef}() -register_to_keeper(x::TArray) = push!(TArrayKeeper, WeakRef(x)) -function copy_tarrays(task1::Task, task2::Task) - filter!(x -> x.value !== nothing, TArrayKeeper) - for wref in TArrayKeeper - ta = wref.value - if ta !== nothing && haskey(ta.data, task1) && !haskey(ta.data, task2) - ta.data[task2] = ta.data[task1] - end - end -end - -# _local_storage - -_get_local_storage(x) = x -function _get_local_storage(x::TArray; copydata=false) - n, d = x.data[current_task()] - copydata || return d - cn = n_copies() - newd = d - if cn > n - newd = deepcopy(d) - _set_local_storage(x, cn, newd) - end - return newd -end - -function _set_local_storage(x::TArray{T, N}, n_copies::Int, d::AbstractArray{T, N}) where {T, N} - x.data[current_task()] = (n_copies, d) -end +getdata(x::TArray) = x.data +tape_copy(x::TArray) = TArray(deepcopy(x.data)) -localize(x) = x -localize(x::AbstractArray) = TArray(x) # Constructors """ tzeros(dims, ...) -Construct a distributed array of zeros. +Construct a TArray of zeros. Trailing arguments are the same as those accepted by `TArray`. ```julia @@ -114,11 +60,8 @@ Array(tz) # convert to 4-element Array{Int64,1}: [0, 0, 0, 0] ``` """ function tzeros(T::Type, dim) - res = TArray{T,length(dim), Array{T, length(dim)}}(); - n = n_copies() - d = zeros(T,dim) - _set_local_storage(res, n, d) - return res + d = zeros(T, dim) + TArray(d) end tzeros(::Type{T}, d1::Integer, drest::Integer...) where T = tzeros(T, convert(Dims, tuple(d1, drest...))) @@ -142,11 +85,8 @@ Array(tz) # convert to 4-element Array{Float64,1}: [9.0 9. ``` """ function tfill(val::Real, dim) - res = TArray{typeof(val), length(dim), Array{typeof(val), length(dim)}}(); - n = n_copies() - d = fill(val,dim) - _set_local_storage(res, n, d) - return res + d = fill(val, dim) + TArray(d) end # @@ -154,39 +94,31 @@ end # function Base.convert(::Type{Array}, x::TArray) - return convert(Array{eltype(x), ndims(x)}, x) + convert(Array{eltype(x), ndims(x)}, x) end function Base.convert(::Type{Array{T, N}}, x::TArray{T, N}) where {T, N} - c = convert(Array{T, N}, deepcopy(_get_local_storage(x))) - return c + convert(Array{T, N}, getdata(x)) end function Base.convert(::Type{TArray}, x::AbstractArray) - return convert(TArray{eltype(x), ndims(x)}, x) + convert(TArray{eltype(x), ndims(x)}, x) end function Base.convert(::Type{TArray{T, N}}, x::AbstractArray{T, N}) where {T, N} - res = TArray{T, N, typeof(x)}() - n = n_copies() - _set_local_storage(res, n, x) - return res + TArray(x) end # # Representation # function Base.show(io::IO, ::MIME"text/plain", x::TArray) - arr = x.data[x.orig_task][2] - @warn "Here shows the originating task's storage, " * - "not the current task's storage. " * - "Please explicitly call show(::TArray) to display the current task's version of a TArray." - show(io, MIME("text/plain"), arr) + show(io, MIME("text/plain"), getdata(x)) end -Base.show(io::IO, x::TArray) = show(io, _get_local_storage(x)) +Base.show(io::IO, x::TArray) = show(io, getdata(x)) function Base.summary(io::IO, x::TArray) - print(io, "Task Local Array: ") - summary(io, _get_local_storage(x)) + print(io, "TArray: ") + summary(io, getdata(x)) end # @@ -195,19 +127,19 @@ end for F in (:size, :iterate, :firstindex, :lastindex, :axes) - @eval Base.$F(a::TArray, args...) = $F(_get_local_storage(a), args...) + @eval Base.$F(a::TArray, args...) = $F(getdata(a), args...) end # # Similarity implementation # -Base.similar(x::TArray, ::Type{T}, dims::Dims) where T = TArray(similar(_get_local_storage(x), T, dims)) +Base.similar(x::TArray, ::Type{T}, dims::Dims) where T = TArray(similar(getdata(x), T, dims)) for op in [:(==), :≈] - @eval Base.$op(x::TArray, y::AbstractArray) = Base.$op(_get_local_storage(x), y) - @eval Base.$op(x::AbstractArray, y::TArray) = Base.$op(x, _get_local_storage(y)) - @eval Base.$op(x::TArray, y::TArray) = Base.$op(_get_local_storage(x), _get_local_storage(y)) + @eval Base.$op(x::TArray, y::AbstractArray) = Base.$op(getdata(x), y) + @eval Base.$op(x::AbstractArray, y::TArray) = Base.$op(x, getdata(y)) + @eval Base.$op(x::TArray, y::TArray) = Base.$op(getdata(x), getdata(y)) end # @@ -216,91 +148,88 @@ end # Indexing Interface Base.@propagate_inbounds function Base.getindex(x::TArray{T, N}, I::Vararg{Int,N}) where {T, N} - return _get_local_storage(x)[I...] + return getdata(x)[I...] end Base.@propagate_inbounds function Base.setindex!(x::TArray{T, N}, e, I::Vararg{Int,N}) where {T, N} - d = _get_local_storage(x; copydata=true) - d[I...] = e + getdata(x)[I...] = e end function Base.push!(x::TArray, e) - d = _get_local_storage(x; copydata=true) - push!(d, e) + push!(getdata(x), e) end function Base.pop!(x::TArray) - d = _get_local_storage(x; copydata=true) - pop!(d) + pop!(getdata(x)) end # Other methods from stdlib Base.view(x::TArray, inds...; kwargs...) = - Base.view(_get_local_storage(x), inds...; kwargs...) |> localize -Base.:-(x::TArray) = (- _get_local_storage(x)) |> localize -Base.transpose(x::TArray) = transpose(_get_local_storage(x)) |> localize -Base.adjoint(x::TArray) = adjoint(_get_local_storage(x)) |> localize -Base.repeat(x::TArray; kw...) = repeat(_get_local_storage(x); kw...) |> localize + Base.view(getdata(x), inds...; kwargs...) |> TArray +Base.:-(x::TArray) = (-getdata(x)) |> TArray +Base.transpose(x::TArray) = transpose(getdata(x)) |> TArray +Base.adjoint(x::TArray) = adjoint(getdata(x)) |> TArray +Base.repeat(x::TArray; kw...) = repeat(getdata(x); kw...) |> TArray Base.hcat(xs::Union{TArray{T,1}, TArray{T,2}}...) where T = - hcat(_get_local_storage.(xs)...) |> localize + hcat(getdata.(xs)...) |> TArray Base.vcat(xs::Union{TArray{T,1}, TArray{T,2}}...) where T = - vcat(_get_local_storage.(xs)...) |> localize + vcat(getdata.(xs)...) |> TArray Base.cat(xs::Union{TArray{T,1}, TArray{T,2}}...; dims) where T = - cat(_get_local_storage.(xs)...; dims = dims) |> localize + cat(getdata.(xs)...; dims = dims) |> TArray -Base.reshape(x::TArray, dims::Union{Colon,Int}...) = reshape(_get_local_storage(x), dims) |> localize +Base.reshape(x::TArray, dims::Union{Colon,Int}...) = reshape(getdata(x), dims) |> TArray Base.reshape(x::TArray, dims::Tuple{Vararg{Union{Int,Colon}}}) = - reshape(_get_local_storage(x), Base._reshape_uncolon(_get_local_storage(x), dims)) |> localize -Base.reshape(x::TArray, dims::Tuple{Vararg{Int}}) = reshape(_get_local_storage(x), dims) |> localize - -Base.permutedims(x::TArray, perm) = permutedims(_get_local_storage(x), perm) |> localize -Base.PermutedDimsArray(x::TArray, perm) = PermutedDimsArray(_get_local_storage(x), perm) |> localize -Base.reverse(x::TArray; dims) = reverse(_get_local_storage(x), dims = dims) |> localize - -Base.sum(x::TArray; dims = :) = sum(_get_local_storage(x), dims = dims) |> localize -Base.sum(f::Union{Function,Type},x::TArray) = sum(f.(_get_local_storage(x))) |> localize -Base.prod(x::TArray; dims=:) = prod(_get_local_storage(x); dims=dims) |> localize -Base.prod(f::Union{Function, Type}, x::TArray) = prod(f.(_get_local_storage(x))) |> localize - -Base.findfirst(x::TArray, args...) = findfirst(_get_local_storage(x), args...) |> localize -Base.maximum(x::TArray; dims = :) = maximum(_get_local_storage(x), dims = dims) |> localize -Base.minimum(x::TArray; dims = :) = minimum(_get_local_storage(x), dims = dims) |> localize - -Base.:/(x::TArray, y::TArray) = _get_local_storage(x) / _get_local_storage(y) |> localize -Base.:/(x::AbstractArray, y::TArray) = x / _get_local_storage(y) |> localize -Base.:/(x::TArray, y::AbstractArray) = _get_local_storage(x) / y |> localize -Base.:\(x::TArray, y::TArray) = _get_local_storage(x) \ _get_local_storage(y) |> localize -Base.:\(x::AbstractArray, y::TArray) = x \ _get_local_storage(y) |> localize -Base.:\(x::TArray, y::AbstractArray) = _get_local_storage(x) \ y |> localize -Base.:*(x::TArray, y::TArray) = _get_local_storage(x) * _get_local_storage(y) |> localize -Base.:*(x::AbstractArray, y::TArray) = x * _get_local_storage(y) |> localize -Base.:*(x::TArray, y::AbstractArray) = _get_local_storage(x) * y |> localize + reshape(getdata(x), Base._reshape_uncolon(getdata(x), dims)) |> TArray +Base.reshape(x::TArray, dims::Tuple{Vararg{Int}}) = reshape(getdata(x), dims) |> TArray + +Base.permutedims(x::TArray, perm) = permutedims(getdata(x), perm) |> TArray +Base.PermutedDimsArray(x::TArray, perm) = PermutedDimsArray(getdata(x), perm) |> TArray +Base.reverse(x::TArray; dims) = reverse(getdata(x), dims = dims) |> TArray + +Base.sum(x::TArray; dims = :) = sum(getdata(x), dims = dims) |> TArray +Base.sum(f::Union{Function,Type},x::TArray) = sum(f.(getdata(x))) |> TArray +Base.prod(x::TArray; dims=:) = prod(getdata(x); dims=dims) |> TArray +Base.prod(f::Union{Function, Type}, x::TArray) = prod(f.(getdata(x))) |> TArray + +Base.findfirst(x::TArray, args...) = findfirst(getdata(x), args...) |> TArray +Base.maximum(x::TArray; dims = :) = maximum(getdata(x), dims = dims) |> TArray +Base.minimum(x::TArray; dims = :) = minimum(getdata(x), dims = dims) |> TArray + +Base.:/(x::TArray, y::TArray) = getdata(x) / getdata(y) |> TArray +Base.:/(x::AbstractArray, y::TArray) = x / getdata(y) |> TArray +Base.:/(x::TArray, y::AbstractArray) = getdata(x) / y |> TArray +Base.:\(x::TArray, y::TArray) = getdata(x) \ getdata(y) |> TArray +Base.:\(x::AbstractArray, y::TArray) = x \ getdata(y) |> TArray +Base.:\(x::TArray, y::AbstractArray) = getdata(x) \ y |> TArray +Base.:*(x::TArray, y::TArray) = getdata(x) * getdata(y) |> TArray +Base.:*(x::AbstractArray, y::TArray) = x * getdata(y) |> TArray +Base.:*(x::TArray, y::AbstractArray) = getdata(x) * y |> TArray # broadcast Base.BroadcastStyle(::Type{<:TArray}) = Broadcast.ArrayStyle{TArray}() -Broadcast.broadcasted(::Broadcast.ArrayStyle{TArray}, f, args...) = f.(_get_local_storage.(args)...) |> localize +Broadcast.broadcasted(::Broadcast.ArrayStyle{TArray}, f, args...) = f.(getdata.(args)...) |> TArray import LinearAlgebra import LinearAlgebra: \, /, inv, det, logdet, logabsdet, norm -LinearAlgebra.inv(x::TArray) = inv(_get_local_storage(x)) |> localize -LinearAlgebra.det(x::TArray) = det(_get_local_storage(x)) |> localize -LinearAlgebra.logdet(x::TArray) = logdet(_get_local_storage(x)) |> localize -LinearAlgebra.logabsdet(x::TArray) = logabsdet(_get_local_storage(x)) |> localize +LinearAlgebra.inv(x::TArray) = inv(getdata(x)) |> TArray +LinearAlgebra.det(x::TArray) = det(getdata(x)) |> TArray +LinearAlgebra.logdet(x::TArray) = logdet(getdata(x)) |> TArray +LinearAlgebra.logabsdet(x::TArray) = logabsdet(getdata(x)) |> TArray LinearAlgebra.norm(x::TArray, p::Real = 2) = - LinearAlgebra.norm(_get_local_storage(x), p) |> localize + LinearAlgebra.norm(getdata(x), p) |> TArray import LinearAlgebra: dot -dot(x::TArray, ys::TArray) = dot(_get_local_storage(x), _get_local_storage(ys)) |> localize -dot(x::AbstractArray, ys::TArray) = dot(x, _get_local_storage(ys)) |> localize -dot(x::TArray, ys::AbstractArray) = dot(_get_local_storage(x), ys) |> localize +dot(x::TArray, ys::TArray) = dot(getdata(x), getdata(ys)) |> TArray +dot(x::AbstractArray, ys::TArray) = dot(x, getdata(ys)) |> TArray +dot(x::TArray, ys::AbstractArray) = dot(getdata(x), ys) |> TArray using Statistics -Statistics.mean(x::TArray; dims = :) = mean(_get_local_storage(x), dims = dims) |> localize -Statistics.std(x::TArray; kw...) = std(_get_local_storage(x), kw...) |> localize +Statistics.mean(x::TArray; dims = :) = mean(getdata(x), dims = dims) |> TArray +Statistics.std(x::TArray; kw...) = std(getdata(x), kw...) |> TArray # TODO # * NNlib diff --git a/src/tref.jl b/src/tref.jl index 0b0af90c..e6cbf876 100644 --- a/src/tref.jl +++ b/src/tref.jl @@ -6,7 +6,7 @@ TRef(x) Implementation of an abstract data structure that -automatically performs copy-on-write after task copying. +automatically performs deepcopy during task copying. Atomic (single-valued) TRef objects must be set or updated by indexing. For example, to access `val = TRef(1)`, you @@ -34,51 +34,30 @@ d = TRef(Dict("A" => 1, 5 => "B")) d["A"] = 10 ``` """ -struct TRef - ref :: Symbol # object_id - orig_task :: Task - TRef() = new(gensym(), current_task()) +mutable struct TRef{T} + val::T end -function TRef(x) - res = TRef(); - n = n_copies() - task_local_storage(res.ref, (n,Ref(x))) - return res -end +Base.get(r::TRef) = r.val +Base.show(io::IO, r::TRef) = Base.show(io::IO, get(r)) +Base.size(r::TRef) = Base.size(get(r)) +Base.ndims(r::TRef) = Base.ndims(get(r)) +tape_copy(r::TRef) = TRef(deepcopy(get(r))) -function Base.getindex(S::TRef, I::Vararg{Any,N}) where {N} - _, d = task_local_storage(S.ref) - return d[][I...] +function Base.getindex(r::TRef, I::Vararg{Any,N}) where {N} + return get(r)[I...] end -function Base.setindex!(S::TRef, x, I::Vararg{Any,N}) where {N} - n, d = task_local_storage(S.ref) - cn = n_copies() - newd = d - if cn > n - # println("[setindex!]: $(S.ref) copying data") - newd = deepcopy(d) - task_local_storage(S.ref, (cn, newd)) - end - - if isa(newd[], Real) - newd[] = x +function Base.setindex!(r::TRef, x, I::Vararg{Any,N}) where {N} + d = get(r) + if isa(d, Real) + r.val = x else - setindex!(newd[], x, I...) + setindex!(d, x, I...) end - return newd[] + return d end -function Base.display(S::TRef) - display("Please use show(::TRef) instead.") -end - -Base.show(io::IO, S::TRef) = Base.show(io::IO, task_local_storage(S.ref)[2][]) -Base.size(S::TRef) = Base.size(task_local_storage(S.ref)[2][]) -Base.ndims(S::TRef) = Base.ndims(task_local_storage(S.ref)[2][]) - -Base.get(S::TRef) = (current_task().storage[S.ref][2][]) # Implements eltype, firstindex, lastindex, and iterate # functions. diff --git a/test/ctask.jl b/test/ctask.jl index c75c70ad..87b9011b 100644 --- a/test/ctask.jl +++ b/test/ctask.jl @@ -3,7 +3,7 @@ @testset "stack allocated objects" begin function f() t = 0 - while true + while t < 4 produce(t) t = 1 + t end @@ -23,7 +23,7 @@ @testset "heap allocated objects" begin function f() t = [0 1 2] - while true + for _ in 1:10 produce(t[1]) t[1] = 1 + t[1] end @@ -44,7 +44,7 @@ @testset "iteration" begin function f() t = 1 - while true + for _ in 1:12 produce(t) t = 1 + t end @@ -68,12 +68,12 @@ @test a == 4:10 end - # Test of `CTaskException`. - @testset "CTaskException" begin + # Test of `Exception`. + @testset "Exception" begin @testset "method error" begin function f() t = 0 - while true + for _ in 1:3 t[3] = 1 produce(t) t = t + 1 @@ -84,16 +84,17 @@ try consume(ctask) catch ex - @test ex isa Libtask.CTaskException - @test ex.task.exception isa MethodError + @test ex isa MethodError + end + if VERSION >= v"1.5" + @test ctask.task.exception isa MethodError end - @test ctask.task.exception isa MethodError end @testset "error test" begin function f() x = 1 - while true + for _ in 1:3 error("error test") produce(x) x += 1 @@ -104,16 +105,17 @@ try consume(ctask) catch ex - @test ex isa Libtask.CTaskException - @test ex.task.exception isa ErrorException + @test ex isa ErrorException + end + if VERSION >= v"1.5" + @test ctask.task.exception isa ErrorException end - @test ctask.task.exception isa ErrorException end @testset "OutOfBounds Test Before" begin function f() x = zeros(2) - while true + for _ in 1:3 x[1] = 1 x[2] = 2 x[3] = 3 @@ -125,16 +127,17 @@ try consume(ctask) catch ex - @test ex isa Libtask.CTaskException - @test ex.task.exception isa BoundsError + @test ex isa BoundsError + end + if VERSION >= v"1.5" + @test ctask.task.exception isa BoundsError end - @test ctask.task.exception isa BoundsError end @testset "OutOfBounds Test After `produce`" begin function f() x = zeros(2) - while true + for _ in 1:3 x[1] = 1 x[2] = 2 produce(x[2]) @@ -147,16 +150,17 @@ try consume(ctask) catch ex - @test ex isa Libtask.CTaskException - @test ex.task.exception isa BoundsError + @test ex isa BoundsError + end + if VERSION >= v"1.5" + @test ctask.task.exception isa BoundsError end - @test ctask.task.exception isa BoundsError end @testset "OutOfBounds Test After `copy`" begin function f() x = zeros(2) - while true + for _ in 1:3 x[1] = 1 x[2] = 2 produce(x[2]) @@ -170,11 +174,12 @@ try consume(ctask2) catch ex - @test ex isa Libtask.CTaskException - @test ex.task.exception isa BoundsError + @test ex isa BoundsError end @test ctask.task.exception === nothing - @test ctask2.task.exception isa BoundsError + if VERSION >= v"1.5" + @test ctask2.task.exception isa BoundsError + end end end end diff --git a/test/tarray.jl b/test/tarray.jl index 17fa44e7..7f1c3fb3 100644 --- a/test/tarray.jl +++ b/test/tarray.jl @@ -55,7 +55,7 @@ @test_throws MethodError TArray{Int,2}(undef, 4) ta3 = TArray{Int, 4}(4, 3, 2, 1) - ta4 = Libtask._get_local_storage(ta3) + ta4 = Libtask.getdata(ta3) @test ta3[3] == ta4[3] ta5 = TArray{Int}(4) @@ -116,14 +116,14 @@ @test repeat(ta, 1, 2) == hcat(ta, ta) - @test ta .+ ta == Libtask._get_local_storage(ta) .+ Libtask._get_local_storage(ta) + @test ta .+ ta == Libtask.getdata(ta) .+ Libtask.getdata(ta) end @testset "task copy" begin function f() t = TArray(Int, 1) t[1] = 0 - while true + for _ in 1:10 produce(t[1]) t[1] t[1] = 1 + t[1] @@ -139,14 +139,14 @@ consume(a) @test consume(ctask) == 2 - @task consume(a) == 4 + @test consume(a) == 4 DATA = Dict{Task, Array}() function g() ta = tzeros(UInt64, 4) for i in 1:4 - ta[i] = hash(Libtask._current_task()) - DATA[Libtask._current_task()] = convert(Array, ta) + ta[i] = hash(current_task()) + DATA[current_task()] = convert(Array, ta) produce(ta[i]) end end @@ -162,15 +162,14 @@ @test consume(ctask) == hash(ctask.task) # index = 3 @test DATA[ctask.task] == [hash(ctask.task), hash(ctask.task), hash(ctask.task), 0] - @test DATA[a.task] == [hash(ctask.task), hash(ctask.task), hash(a.task), - hash(a.task)] + @test DATA[a.task] == [hash(ctask.task), hash(ctask.task), hash(a.task), hash(a.task)] end @testset "Issue: PR-86 (DynamicPPL.jl/pull/261)" begin function f() t = TArray(Int, 1) t[1] = 0 - while true + for _ in 1:4000 produce(t[1]) t[1] t[1] = 1 + t[1] @@ -181,7 +180,7 @@ ctask = CTask(f) ex = try - for _ in 1:10000 + for _ in 1:999 consume(ctask) consume(ctask) a = copy(ctask) diff --git a/test/tref.jl b/test/tref.jl index 22f6948b..e080bef4 100644 --- a/test/tref.jl +++ b/test/tref.jl @@ -4,7 +4,7 @@ function f() t = TRef(1) t[] = 0 - while true + for _ in 1:6 produce(t[]) t[] t[] += 1 @@ -29,7 +29,7 @@ function f() t = TRef(Dict("A" => 1, 5 => "B")) t["A"] = 0 - while true + for _ in 1:6 produce(t["A"]) t["A"] t["A"] += 1