mirror of
https://github.com/pocketpy/pocketpy
synced 2026-03-25 06:30:17 +00:00
Compare commits
16 Commits
e2dfad3fce
...
b12d2bc4cc
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b12d2bc4cc | ||
|
|
01418a7144 | ||
|
|
e81dc95e6f | ||
|
|
178e77fec3 | ||
|
|
076627fb76 | ||
|
|
13884aaadf | ||
|
|
0528f394e3 | ||
|
|
fa5443e9d1 | ||
|
|
ea7097b0eb | ||
|
|
cdbe55b539 | ||
|
|
5b142b21f8 | ||
|
|
3d6c5c60a3 | ||
|
|
6fcdea7ef8 | ||
|
|
0d39a0b3da | ||
|
|
6f79fb0bf3 | ||
|
|
9448214317 |
2
.github/workflows/main.yml
vendored
2
.github/workflows/main.yml
vendored
@ -24,7 +24,7 @@ jobs:
|
||||
run: |
|
||||
python amalgamate.py
|
||||
cd amalgamated
|
||||
cl.exe /std:c11 /utf-8 /Ox /I. pocketpy.c main.c /link /out:pkpy.exe
|
||||
cl.exe /std:c11 /experimental:c11atomics /utf-8 /Ox /I. pocketpy.c main.c /link /out:pkpy.exe
|
||||
build_win32:
|
||||
runs-on: windows-latest
|
||||
steps:
|
||||
|
||||
@ -18,6 +18,7 @@ endif()
|
||||
|
||||
if(MSVC)
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /utf-8 /jumptablerdata /GS-")
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /experimental:c11atomics")
|
||||
add_compile_options(/wd4267 /wd4244)
|
||||
|
||||
if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
|
||||
|
||||
@ -73,7 +73,7 @@ It is safe to use `main` branch in production if CI badge is green.
|
||||
To compile it with your project, these flags must be set:
|
||||
|
||||
+ `--std=c11` flag must be set
|
||||
+ For MSVC, `/utf-8` flag must be set
|
||||
+ For MSVC, `/utf-8` and `/experimental:c11atomics` flag must be set
|
||||
+ `NDEBUG` macro should be defined for release build, or you will get poor performance
|
||||
|
||||
For amalgamated build, run `python amalgamate.py` to generate `pocketpy.c` and `pocketpy.h` in `amalgamated/` directory.
|
||||
|
||||
@ -149,7 +149,7 @@ write_file('amalgamated/pocketpy.h', merge_h_files())
|
||||
shutil.copy("src2/main.c", "amalgamated/main.c")
|
||||
|
||||
if sys.platform in ['linux', 'darwin']:
|
||||
ok = os.system("gcc -o main amalgamated/pocketpy.c amalgamated/main.c -O1 --std=c11 -lm -ldl")
|
||||
ok = os.system("gcc -o main amalgamated/pocketpy.c amalgamated/main.c -O1 --std=c11 -lm -ldl -lpthread")
|
||||
if ok == 0:
|
||||
print("Test build success!")
|
||||
|
||||
|
||||
2
build.sh
2
build.sh
@ -20,7 +20,7 @@ SRC2=${1:-src2/main.c}
|
||||
|
||||
echo "> Compiling and linking source files... "
|
||||
|
||||
clang -std=c11 -O2 -Wfatal-errors -Iinclude -DNDEBUG -o main $SRC $SRC2 -lm -ldl
|
||||
clang -std=c11 -O2 -Wfatal-errors -Iinclude -DNDEBUG -o main $SRC $SRC2 -lm -ldl -lpthread
|
||||
|
||||
if [ $? -eq 0 ]; then
|
||||
echo "Build completed. Type \"./main\" to enter REPL."
|
||||
|
||||
@ -4,7 +4,7 @@ set -e
|
||||
|
||||
SRC=$(find src/ -name "*.c")
|
||||
|
||||
FLAGS="-std=c11 -lm -ldl -Iinclude -O0 -Wfatal-errors -g -DDEBUG -DPK_ENABLE_OS=1"
|
||||
FLAGS="-std=c11 -lm -ldl -lpthread -Iinclude -O0 -Wfatal-errors -g -DDEBUG -DPK_ENABLE_OS=1"
|
||||
|
||||
SANITIZE_FLAGS="-fsanitize=address,leak,undefined"
|
||||
|
||||
|
||||
@ -4,7 +4,7 @@ set -e
|
||||
|
||||
SRC=$(find src/ -name "*.c")
|
||||
|
||||
FLAGS="-std=c11 -lm -ldl -Iinclude -O0 -Wfatal-errors -g -DDEBUG -DPK_ENABLE_OS=1"
|
||||
FLAGS="-std=c11 -lm -ldl -lpthread -Iinclude -O0 -Wfatal-errors -g -DDEBUG -DPK_ENABLE_OS=1"
|
||||
|
||||
SANITIZE_FLAGS="-fsanitize=address,leak,undefined"
|
||||
|
||||
|
||||
153
docs/features/threading.md
Normal file
153
docs/features/threading.md
Normal file
@ -0,0 +1,153 @@
|
||||
---
|
||||
icon: dot
|
||||
title: Threading
|
||||
---
|
||||
|
||||
pocketpy organizes its state by `VM` structure.
|
||||
Users can have at maximum 16 `VM` instances (index from 0 to 15).
|
||||
Each `VM` instance can only be accessed by exactly one thread at a time.
|
||||
If you are trying to run two python scripts in parallel refering the same `VM` instance,
|
||||
you will crash it definitely.
|
||||
|
||||
However, there are two ways to achieve multi-threading in pocketpy.
|
||||
|
||||
One way is to use a native threading library such as `pthread`.
|
||||
You can wrap the multi-threading logic into a C function and bind it to pocketpy.
|
||||
Be careful and not to access the same `VM` instance from multiple threads at the same time.
|
||||
You need to lock critical resources or perform a deep copy of all needed data.
|
||||
|
||||
## ComputeThread
|
||||
|
||||
The other way is to use `pkpy.ComputeThread`.
|
||||
It is like an isolate in Dart language.
|
||||
`ComputeThread` is a true multi-threading model to allow you run python scripts in parallel without lock,
|
||||
backed by a separate `VM` instance.
|
||||
|
||||
`ComputeThread` is highly designed for computational intensive tasks in games.
|
||||
For example, you can run game logic in main thread (VM 0) and run world generation in another thread (e.g. VM 1).
|
||||
|
||||
```mermaid
|
||||
graph TD
|
||||
subgraph Main Thread
|
||||
A[Game Start]
|
||||
B[Submit WorldGen Job]
|
||||
C[Frame 1]
|
||||
D[Frame 2]
|
||||
E[Frame 3]
|
||||
F[...]
|
||||
G[Get WorldGen Result]
|
||||
H[Render World]
|
||||
end
|
||||
subgraph WorldGen Thread
|
||||
O[Generate Biomes]
|
||||
P[Generate Terrain]
|
||||
Q[Generate Creatures]
|
||||
R[Dump Result]
|
||||
end
|
||||
A --> B
|
||||
B --> C
|
||||
C --> D
|
||||
D --> E
|
||||
E --> F
|
||||
F --> G
|
||||
G --> H
|
||||
|
||||
O --> P
|
||||
P --> Q
|
||||
Q --> R
|
||||
|
||||
B --> O
|
||||
R --> G
|
||||
```
|
||||
|
||||
#### `main.py`
|
||||
```python
|
||||
import time
|
||||
from pkpy import ComputeThread
|
||||
|
||||
thread = ComputeThread(1)
|
||||
print("Game Start")
|
||||
|
||||
# import worldgen.py
|
||||
thread.exec('from worldgen import gen_world')
|
||||
|
||||
print("Submit WorldGen Job")
|
||||
thread.submit_call('gen_world', 3, (100, 100), 10)
|
||||
|
||||
# wait for worldgen to finish
|
||||
for i in range(1, 100000):
|
||||
print('Frame:', i)
|
||||
time.sleep(1)
|
||||
if thread.is_done:
|
||||
break
|
||||
|
||||
error = thread.last_error()
|
||||
if error is not None:
|
||||
print("Error:", error)
|
||||
else:
|
||||
retval = thread.last_retval()
|
||||
biomes = retval['biomes']
|
||||
terrain = retval['terrain']
|
||||
creatures = retval['creatures']
|
||||
print("World Generation Complete", len(biomes), len(terrain), len(creatures))
|
||||
```
|
||||
|
||||
#### `worldgen.py`
|
||||
```python
|
||||
import time
|
||||
import random
|
||||
|
||||
def gen_world(biome_count: int, terrain_size: tuple[int, int], creature_count: int) -> dict:
|
||||
# simulate a long computation
|
||||
time.sleep(3)
|
||||
|
||||
# generate world data
|
||||
all_biomes = ["forest", "desert", "ocean", "mountain", "swamp"]
|
||||
all_creatures = ["wolf", "bear", "fish", "bird", "lizard"]
|
||||
|
||||
width, height = terrain_size
|
||||
|
||||
terrain_data = [
|
||||
random.randint(1, 10)
|
||||
for _ in range(width * height)
|
||||
]
|
||||
|
||||
creatures = [
|
||||
{
|
||||
"name": random.choice(all_creatures),
|
||||
"x": random.randint(0, width - 1),
|
||||
"y": random.randint(0, height - 1),
|
||||
}
|
||||
for i in range(creature_count)
|
||||
]
|
||||
|
||||
return {
|
||||
"biomes": all_biomes[:biome_count],
|
||||
"terrain": terrain_data,
|
||||
"creatures": creatures,
|
||||
}
|
||||
```
|
||||
|
||||
Run `main.py` and you will see the result like this:
|
||||
```
|
||||
Game Start
|
||||
Submit WorldGen Job
|
||||
Frame: 1
|
||||
Frame: 2
|
||||
Frame: 3
|
||||
Frame: 4
|
||||
World Generation Complete 3 10000 10
|
||||
```
|
||||
|
||||
`ComputeThread` uses `pickle` module to serialize the data between threads.
|
||||
Parameters and return values must be supported by `pickle`.
|
||||
See [pickle](https://pocketpy.dev/modules/pickle/) for more details.
|
||||
|
||||
Since `ComputeThread` is backed by a separate `VM` instance,
|
||||
it does not share any state with the main thread
|
||||
except for the parameters you pass to it.
|
||||
Therefore, common python modules will be imported twice in each thread.
|
||||
|
||||
If you want to identify which VM instance the module is running in,
|
||||
you can call `pkpy.currentvm` or let your `ComputeThread` set some special flags
|
||||
before importing these modules.
|
||||
10
docs/modules/pkpy.md
Normal file
10
docs/modules/pkpy.md
Normal file
@ -0,0 +1,10 @@
|
||||
---
|
||||
icon: package
|
||||
label: pkpy
|
||||
---
|
||||
|
||||
Provide internal access to the pocketpy interpreter.
|
||||
|
||||
#### Source code
|
||||
|
||||
:::code source="../../include/typings/pkpy.pyi" :::
|
||||
@ -30,7 +30,7 @@ It is safe to use `main` branch in production if CI badge is green.
|
||||
To compile it with your project, these flags must be set:
|
||||
|
||||
+ `--std=c11` flag must be set
|
||||
+ For MSVC, `/utf-8` flag must be set
|
||||
+ For MSVC, `/utf-8` and `/experimental:c11atomics` flag must be set
|
||||
+ `NDEBUG` macro should be defined for release build, or you will get poor performance
|
||||
|
||||
### Get prebuilt binaries
|
||||
|
||||
@ -68,6 +68,9 @@ int c11__u8_header(unsigned char c, bool suppress);
|
||||
int c11__u8_value(int u8bytes, const char* data);
|
||||
int c11__u32_to_u8(uint32_t utf32_char, char utf8_output[4]);
|
||||
|
||||
char* c11_strdup(const char* str);
|
||||
unsigned char* c11_memdup(const unsigned char* data, int size);
|
||||
|
||||
typedef enum IntParsingResult {
|
||||
IntParsing_SUCCESS,
|
||||
IntParsing_FAILURE,
|
||||
|
||||
19
include/pocketpy/common/threads.h
Normal file
19
include/pocketpy/common/threads.h
Normal file
@ -0,0 +1,19 @@
|
||||
#pragma once
|
||||
|
||||
#include <stdatomic.h>
|
||||
#include <stdbool.h>
|
||||
|
||||
#if __EMSCRIPTEN__ || __APPLE__ || __linux__
|
||||
#include <pthread.h>
|
||||
#define PK_USE_PTHREADS 1
|
||||
typedef pthread_t c11_thrd_t;
|
||||
typedef void* c11_thrd_retval_t;
|
||||
#else
|
||||
#include <threads.h>
|
||||
#define PK_USE_PTHREADS 0
|
||||
typedef thrd_t c11_thrd_t;
|
||||
typedef int c11_thrd_retval_t;
|
||||
#endif
|
||||
|
||||
bool c11_thrd_create(c11_thrd_t* thrd, c11_thrd_retval_t (*func)(void*), void* arg);
|
||||
void c11_thrd_yield();
|
||||
@ -591,10 +591,8 @@ class property : public object {
|
||||
object(type::of<property>()(getter, setter)) {}
|
||||
};
|
||||
|
||||
class staticmethod : public object {
|
||||
PKBIND_TYPE_IMPL(object, staticmethod, tp_staticmethod);
|
||||
|
||||
staticmethod(handle method) : object(type::of<staticmethod>()(method)) {}
|
||||
class staticmethod : public cpp_function {
|
||||
PKBIND_TYPE_IMPL(cpp_function, staticmethod, tp_staticmethod);
|
||||
};
|
||||
|
||||
namespace impl {
|
||||
@ -619,8 +617,7 @@ void bind_function(handle obj, const char* name_, Fn&& fn, const Extras&... extr
|
||||
py_setdict(
|
||||
obj.ptr(),
|
||||
name,
|
||||
staticmethod(cpp_function(is_method, name_, std::forward<Fn>(fn), extras...).ptr())
|
||||
.ptr());
|
||||
staticmethod(is_method, name_, std::forward<Fn>(fn), extras...).ptr());
|
||||
} else {
|
||||
if constexpr(has_named_args && is_method) {
|
||||
py_setdict(
|
||||
|
||||
@ -195,6 +195,18 @@ TEST_F(PYBIND11_TEST, overload) {
|
||||
|
||||
EXPECT_EVAL_EQ("cal(1, 2)", 3);
|
||||
EXPECT_EVAL_EQ("cal(1, 2, 3)", 6);
|
||||
|
||||
struct Point {
|
||||
static int sum(int x) { return x; }
|
||||
|
||||
static int sum(int x, int y) { return x + y; }
|
||||
};
|
||||
|
||||
py::class_<Point>(m, "Point")
|
||||
.def_static("sum", py::overload_cast<int>(&Point::sum))
|
||||
.def_static("sum", py::overload_cast<int, int>(&Point::sum));
|
||||
EXPECT_EVAL_EQ("Point.sum(1)", 1);
|
||||
EXPECT_EVAL_EQ("Point.sum(1, 2)", 3);
|
||||
}
|
||||
|
||||
TEST_F(PYBIND11_TEST, return_value_policy) {
|
||||
@ -399,4 +411,3 @@ TEST_F(PYBIND11_TEST, overload_cast) {
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
from typing import Self
|
||||
from typing import Self, Literal
|
||||
from linalg import vec2, vec2i
|
||||
|
||||
class TValue[T]:
|
||||
@ -17,3 +17,35 @@ def memory_usage() -> str:
|
||||
|
||||
def is_user_defined_type(t: type) -> bool:
|
||||
"""Check if a type is user-defined. This means the type was created by executing python `class` statement."""
|
||||
|
||||
def currentvm() -> int:
|
||||
"""Return the current VM index."""
|
||||
|
||||
|
||||
class ComputeThread:
|
||||
def __init__(self, vm_index: Literal[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]): ...
|
||||
|
||||
@property
|
||||
def is_done(self) -> bool:
|
||||
"""Check if the current job is done."""
|
||||
|
||||
def wait_for_done(self) -> None:
|
||||
"""Wait for the current job to finish."""
|
||||
|
||||
def last_error(self) -> str | None: ...
|
||||
def last_retval(self): ...
|
||||
|
||||
def submit_exec(self, source: str) -> None:
|
||||
"""Submit a job to execute some source code."""
|
||||
|
||||
def submit_eval(self, source: str) -> None:
|
||||
"""Submit a job to evaluate some source code."""
|
||||
|
||||
def submit_call(self, eval_src: str, *args, **kwargs) -> None:
|
||||
"""Submit a job to call a function with arguments."""
|
||||
|
||||
def exec(self, source: str) -> None:
|
||||
"""Directly execute some source code."""
|
||||
|
||||
def eval(self, source: str):
|
||||
"""Directly evaluate some source code."""
|
||||
|
||||
@ -348,6 +348,20 @@ int c11__u32_to_u8(uint32_t utf32_char, char utf8_output[4]) {
|
||||
return length;
|
||||
}
|
||||
|
||||
char* c11_strdup(const char* str){
|
||||
int len = strlen(str);
|
||||
char* dst = PK_MALLOC(len + 1);
|
||||
memcpy(dst, str, len);
|
||||
dst[len] = '\0';
|
||||
return dst;
|
||||
}
|
||||
|
||||
unsigned char* c11_memdup(const unsigned char* src, int size) {
|
||||
unsigned char* dst = PK_MALLOC(size);
|
||||
memcpy(dst, src, size);
|
||||
return dst;
|
||||
}
|
||||
|
||||
IntParsingResult c11__parse_uint(c11_sv text, int64_t* out, int base) {
|
||||
*out = 0;
|
||||
|
||||
|
||||
22
src/common/threads.c
Normal file
22
src/common/threads.c
Normal file
@ -0,0 +1,22 @@
|
||||
#include "pocketpy/export.h"
|
||||
#include "pocketpy/common/threads.h"
|
||||
|
||||
#if PK_USE_PTHREADS
|
||||
|
||||
bool c11_thrd_create(c11_thrd_t* thrd, c11_thrd_retval_t (*func)(void*), void* arg) {
|
||||
int res = pthread_create(thrd, NULL, func, arg);
|
||||
return res == 0;
|
||||
}
|
||||
|
||||
void c11_thrd_yield() { sched_yield(); }
|
||||
|
||||
#else
|
||||
|
||||
bool c11_thrd_create(c11_thrd_t* thrd, c11_thrd_retval_t (*func)(void*), void* arg) {
|
||||
int res = thrd_create(thrd, func, arg);
|
||||
return res == thrd_success;
|
||||
}
|
||||
|
||||
void c11_thrd_yield() { thrd_yield(); }
|
||||
|
||||
#endif
|
||||
@ -7,6 +7,8 @@
|
||||
#include "pocketpy/common/sstream.h"
|
||||
#include "pocketpy/interpreter/vm.h"
|
||||
|
||||
#include "pocketpy/common/threads.h"
|
||||
|
||||
#define DEF_TVALUE_METHODS(T, Field) \
|
||||
static bool TValue_##T##__new__(int argc, py_Ref argv) { \
|
||||
PY_CHECK_ARGC(2); \
|
||||
@ -63,6 +65,358 @@ static bool pkpy_is_user_defined_type(int argc, py_Ref argv) {
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool pkpy_currentvm(int argc, py_Ref argv) {
|
||||
PY_CHECK_ARGC(0);
|
||||
py_newint(py_retval(), py_currentvm());
|
||||
return true;
|
||||
}
|
||||
|
||||
typedef struct c11_ComputeThread c11_ComputeThread;
|
||||
|
||||
typedef struct {
|
||||
c11_ComputeThread* self;
|
||||
char* eval_src;
|
||||
unsigned char* args_data;
|
||||
int args_size;
|
||||
unsigned char* kwargs_data;
|
||||
int kwargs_size;
|
||||
} ComputeThreadJobCall;
|
||||
|
||||
typedef struct {
|
||||
c11_ComputeThread* self;
|
||||
char* source;
|
||||
enum py_CompileMode mode;
|
||||
} ComputeThreadJobExec;
|
||||
|
||||
static void ComputeThreadJobCall__dtor(void* arg) {
|
||||
ComputeThreadJobCall* self = arg;
|
||||
PK_FREE(self->eval_src);
|
||||
PK_FREE(self->args_data);
|
||||
PK_FREE(self->kwargs_data);
|
||||
}
|
||||
|
||||
static void ComputeThreadJobExec__dtor(void* arg) {
|
||||
ComputeThreadJobExec* self = arg;
|
||||
PK_FREE(self->source);
|
||||
}
|
||||
|
||||
typedef struct c11_ComputeThread {
|
||||
int vm_index;
|
||||
atomic_bool is_done;
|
||||
unsigned char* last_retval_data;
|
||||
int last_retval_size;
|
||||
char* last_error;
|
||||
|
||||
c11_thrd_t thread;
|
||||
void* job;
|
||||
void (*job_dtor)(void*);
|
||||
} c11_ComputeThread;
|
||||
|
||||
static void
|
||||
c11_ComputeThread__reset_job(c11_ComputeThread* self, void* job, void (*job_dtor)(void*)) {
|
||||
if(self->job) {
|
||||
self->job_dtor(self->job);
|
||||
PK_FREE(self->job);
|
||||
}
|
||||
self->job = job;
|
||||
self->job_dtor = job_dtor;
|
||||
}
|
||||
|
||||
static bool _pk_compute_thread_flags[16];
|
||||
|
||||
static void c11_ComputeThread__dtor(c11_ComputeThread* self) {
|
||||
if(!self->is_done) {
|
||||
c11__abort("ComputeThread(%d) is not done yet!! But the object was deleted.",
|
||||
self->vm_index);
|
||||
}
|
||||
if(self->last_retval_data) PK_FREE(self->last_retval_data);
|
||||
if(self->last_error) PK_FREE(self->last_error);
|
||||
c11_ComputeThread__reset_job(self, NULL, NULL);
|
||||
_pk_compute_thread_flags[self->vm_index] = false;
|
||||
}
|
||||
|
||||
static void c11_ComputeThread__on_job_begin(c11_ComputeThread* self) {
|
||||
if(self->last_retval_data) {
|
||||
PK_FREE(self->last_retval_data);
|
||||
self->last_retval_data = NULL;
|
||||
self->last_retval_size = 0;
|
||||
}
|
||||
if(self->last_error) {
|
||||
PK_FREE(self->last_error);
|
||||
self->last_error = NULL;
|
||||
}
|
||||
py_switchvm(self->vm_index);
|
||||
}
|
||||
|
||||
static bool ComputeThread__new__(int argc, py_Ref argv) {
|
||||
c11_ComputeThread* self =
|
||||
py_newobject(py_retval(), py_totype(argv), 0, sizeof(c11_ComputeThread));
|
||||
self->vm_index = 0;
|
||||
self->is_done = true;
|
||||
self->last_retval_data = NULL;
|
||||
self->last_retval_size = 0;
|
||||
self->last_error = NULL;
|
||||
self->job = NULL;
|
||||
self->job_dtor = NULL;
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool ComputeThread__init__(int argc, py_Ref argv) {
|
||||
PY_CHECK_ARGC(2);
|
||||
PY_CHECK_ARG_TYPE(1, tp_int);
|
||||
c11_ComputeThread* self = py_touserdata(py_arg(0));
|
||||
int index = py_toint(py_arg(1));
|
||||
if(index >= 1 && index < 16) {
|
||||
if(_pk_compute_thread_flags[index]) {
|
||||
return ValueError("vm_index %d is already in use", index);
|
||||
}
|
||||
_pk_compute_thread_flags[index] = true;
|
||||
self->vm_index = index;
|
||||
} else {
|
||||
return ValueError("vm_index %d is out of range", index);
|
||||
}
|
||||
py_newnone(py_retval());
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool ComputeThread_is_done(int argc, py_Ref argv) {
|
||||
PY_CHECK_ARGC(1);
|
||||
c11_ComputeThread* self = py_touserdata(argv);
|
||||
py_newbool(py_retval(), self->is_done);
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool ComputeThread_wait_for_done(int argc, py_Ref argv) {
|
||||
PY_CHECK_ARGC(1);
|
||||
c11_ComputeThread* self = py_touserdata(argv);
|
||||
while(!self->is_done)
|
||||
c11_thrd_yield();
|
||||
py_newnone(py_retval());
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool ComputeThread_last_error(int argc, py_Ref argv) {
|
||||
PY_CHECK_ARGC(1);
|
||||
c11_ComputeThread* self = py_touserdata(argv);
|
||||
if(!self->is_done) return OSError("thread is not done yet");
|
||||
if(self->last_error) {
|
||||
py_newstr(py_retval(), self->last_error);
|
||||
} else {
|
||||
py_newnone(py_retval());
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool ComputeThread_last_retval(int argc, py_Ref argv) {
|
||||
PY_CHECK_ARGC(1);
|
||||
c11_ComputeThread* self = py_touserdata(argv);
|
||||
if(!self->is_done) return OSError("thread is not done yet");
|
||||
if(self->last_retval_data == NULL) return ValueError("no retval available");
|
||||
return py_pickle_loads(self->last_retval_data, self->last_retval_size);
|
||||
}
|
||||
|
||||
static c11_thrd_retval_t ComputeThreadJob_call(void* arg) {
|
||||
ComputeThreadJobCall* job = arg;
|
||||
c11_ComputeThread* self = job->self;
|
||||
c11_ComputeThread__on_job_begin(self);
|
||||
|
||||
py_StackRef p0 = py_peek(0);
|
||||
|
||||
if(!py_pusheval(job->eval_src, NULL)) goto __ERROR;
|
||||
// [callable]
|
||||
if(!py_pickle_loads(job->args_data, job->args_size)) goto __ERROR;
|
||||
py_push(py_retval());
|
||||
// [callable, args]
|
||||
if(!py_pickle_loads(job->kwargs_data, job->kwargs_size)) goto __ERROR;
|
||||
py_push(py_retval());
|
||||
// [callable, args, kwargs]
|
||||
if(!py_smarteval("_0(*_1, **_2)", NULL, py_peek(-3), py_peek(-2), py_peek(-1))) goto __ERROR;
|
||||
|
||||
py_shrink(3);
|
||||
if(!py_pickle_dumps(py_retval())) goto __ERROR;
|
||||
int retval_size;
|
||||
unsigned char* retval_data = py_tobytes(py_retval(), &retval_size);
|
||||
self->last_retval_data = c11_memdup(retval_data, retval_size);
|
||||
self->last_retval_size = retval_size;
|
||||
self->is_done = true;
|
||||
return (c11_thrd_retval_t)0;
|
||||
|
||||
__ERROR:
|
||||
self->last_error = py_formatexc();
|
||||
self->is_done = true;
|
||||
py_clearexc(p0);
|
||||
py_newnone(py_retval());
|
||||
return (c11_thrd_retval_t)0;
|
||||
}
|
||||
|
||||
static c11_thrd_retval_t ComputeThreadJob_exec(void* arg) {
|
||||
ComputeThreadJobExec* job = arg;
|
||||
c11_ComputeThread* self = job->self;
|
||||
c11_ComputeThread__on_job_begin(self);
|
||||
|
||||
py_StackRef p0 = py_peek(0);
|
||||
if(!py_exec(job->source, "<job>", job->mode, NULL)) goto __ERROR;
|
||||
if(!py_pickle_dumps(py_retval())) goto __ERROR;
|
||||
int retval_size;
|
||||
unsigned char* retval_data = py_tobytes(py_retval(), &retval_size);
|
||||
self->last_retval_data = c11_memdup(retval_data, retval_size);
|
||||
self->last_retval_size = retval_size;
|
||||
self->is_done = true;
|
||||
return (c11_thrd_retval_t)0;
|
||||
|
||||
__ERROR:
|
||||
self->last_error = py_formatexc();
|
||||
self->is_done = true;
|
||||
py_clearexc(p0);
|
||||
return (c11_thrd_retval_t)0;
|
||||
}
|
||||
|
||||
static bool ComputeThread_submit_exec(int argc, py_Ref argv) {
|
||||
PY_CHECK_ARGC(2);
|
||||
c11_ComputeThread* self = py_touserdata(py_arg(0));
|
||||
if(!self->is_done) return OSError("thread is not done yet");
|
||||
PY_CHECK_ARG_TYPE(1, tp_str);
|
||||
const char* source = py_tostr(py_arg(1));
|
||||
/**************************/
|
||||
ComputeThreadJobExec* job = PK_MALLOC(sizeof(ComputeThreadJobExec));
|
||||
job->self = self;
|
||||
job->source = c11_strdup(source);
|
||||
job->mode = EXEC_MODE;
|
||||
c11_ComputeThread__reset_job(self, job, ComputeThreadJobExec__dtor);
|
||||
/**************************/
|
||||
self->is_done = false;
|
||||
bool ok = c11_thrd_create(&self->thread, ComputeThreadJob_exec, job);
|
||||
if(!ok) {
|
||||
self->is_done = true;
|
||||
return OSError("thrd_create() failed");
|
||||
}
|
||||
py_newnone(py_retval());
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool ComputeThread_submit_eval(int argc, py_Ref argv) {
|
||||
PY_CHECK_ARGC(2);
|
||||
c11_ComputeThread* self = py_touserdata(py_arg(0));
|
||||
if(!self->is_done) return OSError("thread is not done yet");
|
||||
PY_CHECK_ARG_TYPE(1, tp_str);
|
||||
const char* source = py_tostr(py_arg(1));
|
||||
/**************************/
|
||||
ComputeThreadJobExec* job = PK_MALLOC(sizeof(ComputeThreadJobExec));
|
||||
job->self = self;
|
||||
job->source = c11_strdup(source);
|
||||
job->mode = EVAL_MODE;
|
||||
c11_ComputeThread__reset_job(self, job, ComputeThreadJobExec__dtor);
|
||||
/**************************/
|
||||
self->is_done = false;
|
||||
bool ok = c11_thrd_create(&self->thread, ComputeThreadJob_exec, job);
|
||||
if(!ok) {
|
||||
self->is_done = true;
|
||||
return OSError("thrd_create() failed");
|
||||
}
|
||||
py_newnone(py_retval());
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool ComputeThread_submit_call(int argc, py_Ref argv) {
|
||||
PY_CHECK_ARGC(4);
|
||||
c11_ComputeThread* self = py_touserdata(py_arg(0));
|
||||
if(!self->is_done) return OSError("thread is not done yet");
|
||||
PY_CHECK_ARG_TYPE(1, tp_str);
|
||||
PY_CHECK_ARG_TYPE(2, tp_tuple);
|
||||
PY_CHECK_ARG_TYPE(3, tp_dict);
|
||||
// eval_src
|
||||
const char* eval_src = py_tostr(py_arg(1));
|
||||
// *args
|
||||
if(!py_pickle_dumps(py_arg(2))) return false;
|
||||
int args_size;
|
||||
unsigned char* args_data = py_tobytes(py_retval(), &args_size);
|
||||
// *kwargs
|
||||
if(!py_pickle_dumps(py_arg(3))) return false;
|
||||
int kwargs_size;
|
||||
unsigned char* kwargs_data = py_tobytes(py_retval(), &kwargs_size);
|
||||
/**************************/
|
||||
ComputeThreadJobCall* job = PK_MALLOC(sizeof(ComputeThreadJobCall));
|
||||
job->self = self;
|
||||
job->eval_src = c11_strdup(eval_src);
|
||||
job->args_data = c11_memdup(args_data, args_size);
|
||||
job->args_size = args_size;
|
||||
job->kwargs_data = c11_memdup(kwargs_data, kwargs_size);
|
||||
job->kwargs_size = kwargs_size;
|
||||
c11_ComputeThread__reset_job(self, job, ComputeThreadJobCall__dtor);
|
||||
/**************************/
|
||||
self->is_done = false;
|
||||
bool ok = c11_thrd_create(&self->thread, ComputeThreadJob_call, job);
|
||||
if(!ok) {
|
||||
self->is_done = true;
|
||||
return OSError("thrd_create() failed");
|
||||
}
|
||||
py_newnone(py_retval());
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool c11_ComputeThread__exec_blocked(c11_ComputeThread* self,
|
||||
const char* source,
|
||||
enum py_CompileMode mode) {
|
||||
if(!self->is_done) return OSError("thread is not done yet");
|
||||
self->is_done = false;
|
||||
char* err = NULL;
|
||||
int old_vm_index = py_currentvm();
|
||||
py_switchvm(self->vm_index);
|
||||
py_StackRef p0 = py_peek(0);
|
||||
if(!py_exec(source, "<job_blocked>", mode, NULL)) goto __ERROR;
|
||||
if(!py_pickle_dumps(py_retval())) goto __ERROR;
|
||||
|
||||
int retval_size;
|
||||
unsigned char* retval_data = py_tobytes(py_retval(), &retval_size);
|
||||
py_switchvm(old_vm_index);
|
||||
bool ok = py_pickle_loads(retval_data, retval_size);
|
||||
self->is_done = true;
|
||||
return ok;
|
||||
|
||||
__ERROR:
|
||||
err = py_formatexc();
|
||||
py_clearexc(p0);
|
||||
py_switchvm(old_vm_index);
|
||||
self->is_done = true;
|
||||
RuntimeError("c11_ComputeThread__exec_blocked() failed:\n%s", err);
|
||||
PK_FREE(err);
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool ComputeThread_exec(int argc, py_Ref argv) {
|
||||
PY_CHECK_ARGC(2);
|
||||
c11_ComputeThread* self = py_touserdata(py_arg(0));
|
||||
PY_CHECK_ARG_TYPE(1, tp_str);
|
||||
const char* source = py_tostr(py_arg(1));
|
||||
return c11_ComputeThread__exec_blocked(self, source, EXEC_MODE);
|
||||
}
|
||||
|
||||
static bool ComputeThread_eval(int argc, py_Ref argv) {
|
||||
PY_CHECK_ARGC(2);
|
||||
c11_ComputeThread* self = py_touserdata(py_arg(0));
|
||||
PY_CHECK_ARG_TYPE(1, tp_str);
|
||||
const char* source = py_tostr(py_arg(1));
|
||||
return c11_ComputeThread__exec_blocked(self, source, EVAL_MODE);
|
||||
}
|
||||
|
||||
static void pk_ComputeThread__register(py_Ref mod) {
|
||||
py_Type type = py_newtype("ComputeThread", tp_object, mod, (py_Dtor)c11_ComputeThread__dtor);
|
||||
|
||||
py_bindmagic(type, __new__, ComputeThread__new__);
|
||||
py_bindmagic(type, __init__, ComputeThread__init__);
|
||||
py_bindproperty(type, "is_done", ComputeThread_is_done, NULL);
|
||||
py_bindmethod(type, "wait_for_done", ComputeThread_wait_for_done);
|
||||
py_bindmethod(type, "last_error", ComputeThread_last_error);
|
||||
py_bindmethod(type, "last_retval", ComputeThread_last_retval);
|
||||
|
||||
py_bindmethod(type, "submit_exec", ComputeThread_submit_exec);
|
||||
py_bindmethod(type, "submit_eval", ComputeThread_submit_eval);
|
||||
py_bind(py_tpobject(type), "submit_call(self, eval_src, *args, **kwargs)", ComputeThread_submit_call);
|
||||
|
||||
py_bindmethod(type, "exec", ComputeThread_exec);
|
||||
py_bindmethod(type, "eval", ComputeThread_eval);
|
||||
}
|
||||
|
||||
void pk__add_module_pkpy() {
|
||||
py_Ref mod = py_newmodule("pkpy");
|
||||
|
||||
@ -99,6 +453,10 @@ void pk__add_module_pkpy() {
|
||||
|
||||
py_bindfunc(mod, "memory_usage", pkpy_memory_usage);
|
||||
py_bindfunc(mod, "is_user_defined_type", pkpy_is_user_defined_type);
|
||||
|
||||
py_bindfunc(mod, "currentvm", pkpy_currentvm);
|
||||
|
||||
pk_ComputeThread__register(mod);
|
||||
}
|
||||
|
||||
#undef DEF_TVALUE_METHODS
|
||||
49
src2/multi_vm_isolate.c
Normal file
49
src2/multi_vm_isolate.c
Normal file
@ -0,0 +1,49 @@
|
||||
#include "pocketpy.h"
|
||||
#include "threads.h"
|
||||
#include <stdio.h>
|
||||
|
||||
int run_huge_job_in_vm1(void* arg) {
|
||||
py_switchvm(1);
|
||||
bool ok = py_exec((const char*)arg, "<job>", EXEC_MODE, NULL);
|
||||
if(!ok) {
|
||||
py_printexc();
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int main() {
|
||||
py_initialize();
|
||||
|
||||
bool ok = py_exec("print('Hello world from VM0!')", "<string1>", EXEC_MODE, NULL);
|
||||
if(!ok) {
|
||||
py_printexc();
|
||||
return 1;
|
||||
}
|
||||
|
||||
printf("main vm index: %d\n", py_currentvm());
|
||||
|
||||
char* job_string =
|
||||
"import time\n"
|
||||
"res = 0\n"
|
||||
"time.sleep(3)\n"
|
||||
"res = 100\n"
|
||||
"print('Huge job done!')\n"
|
||||
"print('Result:', res)\n";
|
||||
|
||||
thrd_t thread1;
|
||||
thrd_create(&thread1, run_huge_job_in_vm1, job_string);
|
||||
|
||||
for(int i = 0; i < 5; i++) {
|
||||
thrd_sleep(&(struct timespec){.tv_sec = 1, .tv_nsec = 0}, NULL);
|
||||
printf("main vm index: %d\n", py_currentvm());
|
||||
}
|
||||
|
||||
int thrd_res;
|
||||
thrd_join(thread1, &thrd_res);
|
||||
printf("Thread result: %d\n", thrd_res);
|
||||
|
||||
py_finalize();
|
||||
|
||||
return 0;
|
||||
}
|
||||
33
tests/98_thread.py
Normal file
33
tests/98_thread.py
Normal file
@ -0,0 +1,33 @@
|
||||
from pkpy import ComputeThread
|
||||
import time
|
||||
|
||||
thread_1 = ComputeThread(1)
|
||||
thread_2 = ComputeThread(2)
|
||||
|
||||
for t in [thread_1, thread_2]:
|
||||
t.exec('''
|
||||
def func(a):
|
||||
from pkpy import currentvm
|
||||
print("Hello from thread", currentvm(), "a =", a)
|
||||
for i in range(500000):
|
||||
if i % 100000 == 0:
|
||||
print(i, "from thread", currentvm())
|
||||
return a
|
||||
|
||||
x = 123
|
||||
''')
|
||||
assert thread_1.eval('x') == 123
|
||||
|
||||
# thread_1.wait_for_done()
|
||||
# thread_2.wait_for_done()
|
||||
|
||||
thread_1.submit_call('func', [1, 2, 3])
|
||||
thread_2.submit_call('func', [4, 5, 6])
|
||||
|
||||
while not thread_1.is_done or not thread_2.is_done:
|
||||
print("Waiting for threads to finish...")
|
||||
time.sleep(1)
|
||||
|
||||
print("Thread 1 last return value:", thread_1.last_retval())
|
||||
print("Thread 2 last return value:", thread_2.last_retval())
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user