mirror of
https://github.com/pocketpy/pocketpy
synced 2026-03-25 06:30:17 +00:00
Compare commits
No commits in common. "b12d2bc4ccf9b356e407e9162c4b8bde3949f8a3" and "e2dfad3fcebe041eae75cd3db1f6aa41f77971ab" have entirely different histories.
b12d2bc4cc
...
e2dfad3fce
2
.github/workflows/main.yml
vendored
2
.github/workflows/main.yml
vendored
@ -24,7 +24,7 @@ jobs:
|
|||||||
run: |
|
run: |
|
||||||
python amalgamate.py
|
python amalgamate.py
|
||||||
cd amalgamated
|
cd amalgamated
|
||||||
cl.exe /std:c11 /experimental:c11atomics /utf-8 /Ox /I. pocketpy.c main.c /link /out:pkpy.exe
|
cl.exe /std:c11 /utf-8 /Ox /I. pocketpy.c main.c /link /out:pkpy.exe
|
||||||
build_win32:
|
build_win32:
|
||||||
runs-on: windows-latest
|
runs-on: windows-latest
|
||||||
steps:
|
steps:
|
||||||
|
|||||||
@ -18,7 +18,6 @@ endif()
|
|||||||
|
|
||||||
if(MSVC)
|
if(MSVC)
|
||||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /utf-8 /jumptablerdata /GS-")
|
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /utf-8 /jumptablerdata /GS-")
|
||||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /experimental:c11atomics")
|
|
||||||
add_compile_options(/wd4267 /wd4244)
|
add_compile_options(/wd4267 /wd4244)
|
||||||
|
|
||||||
if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
|
if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
|
||||||
|
|||||||
@ -73,7 +73,7 @@ It is safe to use `main` branch in production if CI badge is green.
|
|||||||
To compile it with your project, these flags must be set:
|
To compile it with your project, these flags must be set:
|
||||||
|
|
||||||
+ `--std=c11` flag must be set
|
+ `--std=c11` flag must be set
|
||||||
+ For MSVC, `/utf-8` and `/experimental:c11atomics` flag must be set
|
+ For MSVC, `/utf-8` flag must be set
|
||||||
+ `NDEBUG` macro should be defined for release build, or you will get poor performance
|
+ `NDEBUG` macro should be defined for release build, or you will get poor performance
|
||||||
|
|
||||||
For amalgamated build, run `python amalgamate.py` to generate `pocketpy.c` and `pocketpy.h` in `amalgamated/` directory.
|
For amalgamated build, run `python amalgamate.py` to generate `pocketpy.c` and `pocketpy.h` in `amalgamated/` directory.
|
||||||
|
|||||||
@ -149,7 +149,7 @@ write_file('amalgamated/pocketpy.h', merge_h_files())
|
|||||||
shutil.copy("src2/main.c", "amalgamated/main.c")
|
shutil.copy("src2/main.c", "amalgamated/main.c")
|
||||||
|
|
||||||
if sys.platform in ['linux', 'darwin']:
|
if sys.platform in ['linux', 'darwin']:
|
||||||
ok = os.system("gcc -o main amalgamated/pocketpy.c amalgamated/main.c -O1 --std=c11 -lm -ldl -lpthread")
|
ok = os.system("gcc -o main amalgamated/pocketpy.c amalgamated/main.c -O1 --std=c11 -lm -ldl")
|
||||||
if ok == 0:
|
if ok == 0:
|
||||||
print("Test build success!")
|
print("Test build success!")
|
||||||
|
|
||||||
|
|||||||
2
build.sh
2
build.sh
@ -20,7 +20,7 @@ SRC2=${1:-src2/main.c}
|
|||||||
|
|
||||||
echo "> Compiling and linking source files... "
|
echo "> Compiling and linking source files... "
|
||||||
|
|
||||||
clang -std=c11 -O2 -Wfatal-errors -Iinclude -DNDEBUG -o main $SRC $SRC2 -lm -ldl -lpthread
|
clang -std=c11 -O2 -Wfatal-errors -Iinclude -DNDEBUG -o main $SRC $SRC2 -lm -ldl
|
||||||
|
|
||||||
if [ $? -eq 0 ]; then
|
if [ $? -eq 0 ]; then
|
||||||
echo "Build completed. Type \"./main\" to enter REPL."
|
echo "Build completed. Type \"./main\" to enter REPL."
|
||||||
|
|||||||
@ -4,7 +4,7 @@ set -e
|
|||||||
|
|
||||||
SRC=$(find src/ -name "*.c")
|
SRC=$(find src/ -name "*.c")
|
||||||
|
|
||||||
FLAGS="-std=c11 -lm -ldl -lpthread -Iinclude -O0 -Wfatal-errors -g -DDEBUG -DPK_ENABLE_OS=1"
|
FLAGS="-std=c11 -lm -ldl -Iinclude -O0 -Wfatal-errors -g -DDEBUG -DPK_ENABLE_OS=1"
|
||||||
|
|
||||||
SANITIZE_FLAGS="-fsanitize=address,leak,undefined"
|
SANITIZE_FLAGS="-fsanitize=address,leak,undefined"
|
||||||
|
|
||||||
|
|||||||
@ -4,7 +4,7 @@ set -e
|
|||||||
|
|
||||||
SRC=$(find src/ -name "*.c")
|
SRC=$(find src/ -name "*.c")
|
||||||
|
|
||||||
FLAGS="-std=c11 -lm -ldl -lpthread -Iinclude -O0 -Wfatal-errors -g -DDEBUG -DPK_ENABLE_OS=1"
|
FLAGS="-std=c11 -lm -ldl -Iinclude -O0 -Wfatal-errors -g -DDEBUG -DPK_ENABLE_OS=1"
|
||||||
|
|
||||||
SANITIZE_FLAGS="-fsanitize=address,leak,undefined"
|
SANITIZE_FLAGS="-fsanitize=address,leak,undefined"
|
||||||
|
|
||||||
|
|||||||
@ -1,153 +0,0 @@
|
|||||||
---
|
|
||||||
icon: dot
|
|
||||||
title: Threading
|
|
||||||
---
|
|
||||||
|
|
||||||
pocketpy organizes its state by `VM` structure.
|
|
||||||
Users can have at maximum 16 `VM` instances (index from 0 to 15).
|
|
||||||
Each `VM` instance can only be accessed by exactly one thread at a time.
|
|
||||||
If you are trying to run two python scripts in parallel refering the same `VM` instance,
|
|
||||||
you will crash it definitely.
|
|
||||||
|
|
||||||
However, there are two ways to achieve multi-threading in pocketpy.
|
|
||||||
|
|
||||||
One way is to use a native threading library such as `pthread`.
|
|
||||||
You can wrap the multi-threading logic into a C function and bind it to pocketpy.
|
|
||||||
Be careful and not to access the same `VM` instance from multiple threads at the same time.
|
|
||||||
You need to lock critical resources or perform a deep copy of all needed data.
|
|
||||||
|
|
||||||
## ComputeThread
|
|
||||||
|
|
||||||
The other way is to use `pkpy.ComputeThread`.
|
|
||||||
It is like an isolate in Dart language.
|
|
||||||
`ComputeThread` is a true multi-threading model to allow you run python scripts in parallel without lock,
|
|
||||||
backed by a separate `VM` instance.
|
|
||||||
|
|
||||||
`ComputeThread` is highly designed for computational intensive tasks in games.
|
|
||||||
For example, you can run game logic in main thread (VM 0) and run world generation in another thread (e.g. VM 1).
|
|
||||||
|
|
||||||
```mermaid
|
|
||||||
graph TD
|
|
||||||
subgraph Main Thread
|
|
||||||
A[Game Start]
|
|
||||||
B[Submit WorldGen Job]
|
|
||||||
C[Frame 1]
|
|
||||||
D[Frame 2]
|
|
||||||
E[Frame 3]
|
|
||||||
F[...]
|
|
||||||
G[Get WorldGen Result]
|
|
||||||
H[Render World]
|
|
||||||
end
|
|
||||||
subgraph WorldGen Thread
|
|
||||||
O[Generate Biomes]
|
|
||||||
P[Generate Terrain]
|
|
||||||
Q[Generate Creatures]
|
|
||||||
R[Dump Result]
|
|
||||||
end
|
|
||||||
A --> B
|
|
||||||
B --> C
|
|
||||||
C --> D
|
|
||||||
D --> E
|
|
||||||
E --> F
|
|
||||||
F --> G
|
|
||||||
G --> H
|
|
||||||
|
|
||||||
O --> P
|
|
||||||
P --> Q
|
|
||||||
Q --> R
|
|
||||||
|
|
||||||
B --> O
|
|
||||||
R --> G
|
|
||||||
```
|
|
||||||
|
|
||||||
#### `main.py`
|
|
||||||
```python
|
|
||||||
import time
|
|
||||||
from pkpy import ComputeThread
|
|
||||||
|
|
||||||
thread = ComputeThread(1)
|
|
||||||
print("Game Start")
|
|
||||||
|
|
||||||
# import worldgen.py
|
|
||||||
thread.exec('from worldgen import gen_world')
|
|
||||||
|
|
||||||
print("Submit WorldGen Job")
|
|
||||||
thread.submit_call('gen_world', 3, (100, 100), 10)
|
|
||||||
|
|
||||||
# wait for worldgen to finish
|
|
||||||
for i in range(1, 100000):
|
|
||||||
print('Frame:', i)
|
|
||||||
time.sleep(1)
|
|
||||||
if thread.is_done:
|
|
||||||
break
|
|
||||||
|
|
||||||
error = thread.last_error()
|
|
||||||
if error is not None:
|
|
||||||
print("Error:", error)
|
|
||||||
else:
|
|
||||||
retval = thread.last_retval()
|
|
||||||
biomes = retval['biomes']
|
|
||||||
terrain = retval['terrain']
|
|
||||||
creatures = retval['creatures']
|
|
||||||
print("World Generation Complete", len(biomes), len(terrain), len(creatures))
|
|
||||||
```
|
|
||||||
|
|
||||||
#### `worldgen.py`
|
|
||||||
```python
|
|
||||||
import time
|
|
||||||
import random
|
|
||||||
|
|
||||||
def gen_world(biome_count: int, terrain_size: tuple[int, int], creature_count: int) -> dict:
|
|
||||||
# simulate a long computation
|
|
||||||
time.sleep(3)
|
|
||||||
|
|
||||||
# generate world data
|
|
||||||
all_biomes = ["forest", "desert", "ocean", "mountain", "swamp"]
|
|
||||||
all_creatures = ["wolf", "bear", "fish", "bird", "lizard"]
|
|
||||||
|
|
||||||
width, height = terrain_size
|
|
||||||
|
|
||||||
terrain_data = [
|
|
||||||
random.randint(1, 10)
|
|
||||||
for _ in range(width * height)
|
|
||||||
]
|
|
||||||
|
|
||||||
creatures = [
|
|
||||||
{
|
|
||||||
"name": random.choice(all_creatures),
|
|
||||||
"x": random.randint(0, width - 1),
|
|
||||||
"y": random.randint(0, height - 1),
|
|
||||||
}
|
|
||||||
for i in range(creature_count)
|
|
||||||
]
|
|
||||||
|
|
||||||
return {
|
|
||||||
"biomes": all_biomes[:biome_count],
|
|
||||||
"terrain": terrain_data,
|
|
||||||
"creatures": creatures,
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Run `main.py` and you will see the result like this:
|
|
||||||
```
|
|
||||||
Game Start
|
|
||||||
Submit WorldGen Job
|
|
||||||
Frame: 1
|
|
||||||
Frame: 2
|
|
||||||
Frame: 3
|
|
||||||
Frame: 4
|
|
||||||
World Generation Complete 3 10000 10
|
|
||||||
```
|
|
||||||
|
|
||||||
`ComputeThread` uses `pickle` module to serialize the data between threads.
|
|
||||||
Parameters and return values must be supported by `pickle`.
|
|
||||||
See [pickle](https://pocketpy.dev/modules/pickle/) for more details.
|
|
||||||
|
|
||||||
Since `ComputeThread` is backed by a separate `VM` instance,
|
|
||||||
it does not share any state with the main thread
|
|
||||||
except for the parameters you pass to it.
|
|
||||||
Therefore, common python modules will be imported twice in each thread.
|
|
||||||
|
|
||||||
If you want to identify which VM instance the module is running in,
|
|
||||||
you can call `pkpy.currentvm` or let your `ComputeThread` set some special flags
|
|
||||||
before importing these modules.
|
|
||||||
@ -1,10 +0,0 @@
|
|||||||
---
|
|
||||||
icon: package
|
|
||||||
label: pkpy
|
|
||||||
---
|
|
||||||
|
|
||||||
Provide internal access to the pocketpy interpreter.
|
|
||||||
|
|
||||||
#### Source code
|
|
||||||
|
|
||||||
:::code source="../../include/typings/pkpy.pyi" :::
|
|
||||||
@ -30,7 +30,7 @@ It is safe to use `main` branch in production if CI badge is green.
|
|||||||
To compile it with your project, these flags must be set:
|
To compile it with your project, these flags must be set:
|
||||||
|
|
||||||
+ `--std=c11` flag must be set
|
+ `--std=c11` flag must be set
|
||||||
+ For MSVC, `/utf-8` and `/experimental:c11atomics` flag must be set
|
+ For MSVC, `/utf-8` flag must be set
|
||||||
+ `NDEBUG` macro should be defined for release build, or you will get poor performance
|
+ `NDEBUG` macro should be defined for release build, or you will get poor performance
|
||||||
|
|
||||||
### Get prebuilt binaries
|
### Get prebuilt binaries
|
||||||
|
|||||||
@ -68,9 +68,6 @@ int c11__u8_header(unsigned char c, bool suppress);
|
|||||||
int c11__u8_value(int u8bytes, const char* data);
|
int c11__u8_value(int u8bytes, const char* data);
|
||||||
int c11__u32_to_u8(uint32_t utf32_char, char utf8_output[4]);
|
int c11__u32_to_u8(uint32_t utf32_char, char utf8_output[4]);
|
||||||
|
|
||||||
char* c11_strdup(const char* str);
|
|
||||||
unsigned char* c11_memdup(const unsigned char* data, int size);
|
|
||||||
|
|
||||||
typedef enum IntParsingResult {
|
typedef enum IntParsingResult {
|
||||||
IntParsing_SUCCESS,
|
IntParsing_SUCCESS,
|
||||||
IntParsing_FAILURE,
|
IntParsing_FAILURE,
|
||||||
|
|||||||
@ -1,19 +0,0 @@
|
|||||||
#pragma once
|
|
||||||
|
|
||||||
#include <stdatomic.h>
|
|
||||||
#include <stdbool.h>
|
|
||||||
|
|
||||||
#if __EMSCRIPTEN__ || __APPLE__ || __linux__
|
|
||||||
#include <pthread.h>
|
|
||||||
#define PK_USE_PTHREADS 1
|
|
||||||
typedef pthread_t c11_thrd_t;
|
|
||||||
typedef void* c11_thrd_retval_t;
|
|
||||||
#else
|
|
||||||
#include <threads.h>
|
|
||||||
#define PK_USE_PTHREADS 0
|
|
||||||
typedef thrd_t c11_thrd_t;
|
|
||||||
typedef int c11_thrd_retval_t;
|
|
||||||
#endif
|
|
||||||
|
|
||||||
bool c11_thrd_create(c11_thrd_t* thrd, c11_thrd_retval_t (*func)(void*), void* arg);
|
|
||||||
void c11_thrd_yield();
|
|
||||||
@ -591,8 +591,10 @@ class property : public object {
|
|||||||
object(type::of<property>()(getter, setter)) {}
|
object(type::of<property>()(getter, setter)) {}
|
||||||
};
|
};
|
||||||
|
|
||||||
class staticmethod : public cpp_function {
|
class staticmethod : public object {
|
||||||
PKBIND_TYPE_IMPL(cpp_function, staticmethod, tp_staticmethod);
|
PKBIND_TYPE_IMPL(object, staticmethod, tp_staticmethod);
|
||||||
|
|
||||||
|
staticmethod(handle method) : object(type::of<staticmethod>()(method)) {}
|
||||||
};
|
};
|
||||||
|
|
||||||
namespace impl {
|
namespace impl {
|
||||||
@ -617,7 +619,8 @@ void bind_function(handle obj, const char* name_, Fn&& fn, const Extras&... extr
|
|||||||
py_setdict(
|
py_setdict(
|
||||||
obj.ptr(),
|
obj.ptr(),
|
||||||
name,
|
name,
|
||||||
staticmethod(is_method, name_, std::forward<Fn>(fn), extras...).ptr());
|
staticmethod(cpp_function(is_method, name_, std::forward<Fn>(fn), extras...).ptr())
|
||||||
|
.ptr());
|
||||||
} else {
|
} else {
|
||||||
if constexpr(has_named_args && is_method) {
|
if constexpr(has_named_args && is_method) {
|
||||||
py_setdict(
|
py_setdict(
|
||||||
|
|||||||
@ -195,18 +195,6 @@ TEST_F(PYBIND11_TEST, overload) {
|
|||||||
|
|
||||||
EXPECT_EVAL_EQ("cal(1, 2)", 3);
|
EXPECT_EVAL_EQ("cal(1, 2)", 3);
|
||||||
EXPECT_EVAL_EQ("cal(1, 2, 3)", 6);
|
EXPECT_EVAL_EQ("cal(1, 2, 3)", 6);
|
||||||
|
|
||||||
struct Point {
|
|
||||||
static int sum(int x) { return x; }
|
|
||||||
|
|
||||||
static int sum(int x, int y) { return x + y; }
|
|
||||||
};
|
|
||||||
|
|
||||||
py::class_<Point>(m, "Point")
|
|
||||||
.def_static("sum", py::overload_cast<int>(&Point::sum))
|
|
||||||
.def_static("sum", py::overload_cast<int, int>(&Point::sum));
|
|
||||||
EXPECT_EVAL_EQ("Point.sum(1)", 1);
|
|
||||||
EXPECT_EVAL_EQ("Point.sum(1, 2)", 3);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_F(PYBIND11_TEST, return_value_policy) {
|
TEST_F(PYBIND11_TEST, return_value_policy) {
|
||||||
@ -411,3 +399,4 @@ TEST_F(PYBIND11_TEST, overload_cast) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
} // namespace
|
} // namespace
|
||||||
|
|
||||||
|
|||||||
@ -1,4 +1,4 @@
|
|||||||
from typing import Self, Literal
|
from typing import Self
|
||||||
from linalg import vec2, vec2i
|
from linalg import vec2, vec2i
|
||||||
|
|
||||||
class TValue[T]:
|
class TValue[T]:
|
||||||
@ -16,36 +16,4 @@ def memory_usage() -> str:
|
|||||||
"""Return a summary of the memory usage."""
|
"""Return a summary of the memory usage."""
|
||||||
|
|
||||||
def is_user_defined_type(t: type) -> bool:
|
def is_user_defined_type(t: type) -> bool:
|
||||||
"""Check if a type is user-defined. This means the type was created by executing python `class` statement."""
|
"""Check if a type is user-defined. This means the type was created by executing python `class` statement."""
|
||||||
|
|
||||||
def currentvm() -> int:
|
|
||||||
"""Return the current VM index."""
|
|
||||||
|
|
||||||
|
|
||||||
class ComputeThread:
|
|
||||||
def __init__(self, vm_index: Literal[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]): ...
|
|
||||||
|
|
||||||
@property
|
|
||||||
def is_done(self) -> bool:
|
|
||||||
"""Check if the current job is done."""
|
|
||||||
|
|
||||||
def wait_for_done(self) -> None:
|
|
||||||
"""Wait for the current job to finish."""
|
|
||||||
|
|
||||||
def last_error(self) -> str | None: ...
|
|
||||||
def last_retval(self): ...
|
|
||||||
|
|
||||||
def submit_exec(self, source: str) -> None:
|
|
||||||
"""Submit a job to execute some source code."""
|
|
||||||
|
|
||||||
def submit_eval(self, source: str) -> None:
|
|
||||||
"""Submit a job to evaluate some source code."""
|
|
||||||
|
|
||||||
def submit_call(self, eval_src: str, *args, **kwargs) -> None:
|
|
||||||
"""Submit a job to call a function with arguments."""
|
|
||||||
|
|
||||||
def exec(self, source: str) -> None:
|
|
||||||
"""Directly execute some source code."""
|
|
||||||
|
|
||||||
def eval(self, source: str):
|
|
||||||
"""Directly evaluate some source code."""
|
|
||||||
@ -348,20 +348,6 @@ int c11__u32_to_u8(uint32_t utf32_char, char utf8_output[4]) {
|
|||||||
return length;
|
return length;
|
||||||
}
|
}
|
||||||
|
|
||||||
char* c11_strdup(const char* str){
|
|
||||||
int len = strlen(str);
|
|
||||||
char* dst = PK_MALLOC(len + 1);
|
|
||||||
memcpy(dst, str, len);
|
|
||||||
dst[len] = '\0';
|
|
||||||
return dst;
|
|
||||||
}
|
|
||||||
|
|
||||||
unsigned char* c11_memdup(const unsigned char* src, int size) {
|
|
||||||
unsigned char* dst = PK_MALLOC(size);
|
|
||||||
memcpy(dst, src, size);
|
|
||||||
return dst;
|
|
||||||
}
|
|
||||||
|
|
||||||
IntParsingResult c11__parse_uint(c11_sv text, int64_t* out, int base) {
|
IntParsingResult c11__parse_uint(c11_sv text, int64_t* out, int base) {
|
||||||
*out = 0;
|
*out = 0;
|
||||||
|
|
||||||
|
|||||||
@ -1,22 +0,0 @@
|
|||||||
#include "pocketpy/export.h"
|
|
||||||
#include "pocketpy/common/threads.h"
|
|
||||||
|
|
||||||
#if PK_USE_PTHREADS
|
|
||||||
|
|
||||||
bool c11_thrd_create(c11_thrd_t* thrd, c11_thrd_retval_t (*func)(void*), void* arg) {
|
|
||||||
int res = pthread_create(thrd, NULL, func, arg);
|
|
||||||
return res == 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
void c11_thrd_yield() { sched_yield(); }
|
|
||||||
|
|
||||||
#else
|
|
||||||
|
|
||||||
bool c11_thrd_create(c11_thrd_t* thrd, c11_thrd_retval_t (*func)(void*), void* arg) {
|
|
||||||
int res = thrd_create(thrd, func, arg);
|
|
||||||
return res == thrd_success;
|
|
||||||
}
|
|
||||||
|
|
||||||
void c11_thrd_yield() { thrd_yield(); }
|
|
||||||
|
|
||||||
#endif
|
|
||||||
@ -7,8 +7,6 @@
|
|||||||
#include "pocketpy/common/sstream.h"
|
#include "pocketpy/common/sstream.h"
|
||||||
#include "pocketpy/interpreter/vm.h"
|
#include "pocketpy/interpreter/vm.h"
|
||||||
|
|
||||||
#include "pocketpy/common/threads.h"
|
|
||||||
|
|
||||||
#define DEF_TVALUE_METHODS(T, Field) \
|
#define DEF_TVALUE_METHODS(T, Field) \
|
||||||
static bool TValue_##T##__new__(int argc, py_Ref argv) { \
|
static bool TValue_##T##__new__(int argc, py_Ref argv) { \
|
||||||
PY_CHECK_ARGC(2); \
|
PY_CHECK_ARGC(2); \
|
||||||
@ -65,358 +63,6 @@ static bool pkpy_is_user_defined_type(int argc, py_Ref argv) {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool pkpy_currentvm(int argc, py_Ref argv) {
|
|
||||||
PY_CHECK_ARGC(0);
|
|
||||||
py_newint(py_retval(), py_currentvm());
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
typedef struct c11_ComputeThread c11_ComputeThread;
|
|
||||||
|
|
||||||
typedef struct {
|
|
||||||
c11_ComputeThread* self;
|
|
||||||
char* eval_src;
|
|
||||||
unsigned char* args_data;
|
|
||||||
int args_size;
|
|
||||||
unsigned char* kwargs_data;
|
|
||||||
int kwargs_size;
|
|
||||||
} ComputeThreadJobCall;
|
|
||||||
|
|
||||||
typedef struct {
|
|
||||||
c11_ComputeThread* self;
|
|
||||||
char* source;
|
|
||||||
enum py_CompileMode mode;
|
|
||||||
} ComputeThreadJobExec;
|
|
||||||
|
|
||||||
static void ComputeThreadJobCall__dtor(void* arg) {
|
|
||||||
ComputeThreadJobCall* self = arg;
|
|
||||||
PK_FREE(self->eval_src);
|
|
||||||
PK_FREE(self->args_data);
|
|
||||||
PK_FREE(self->kwargs_data);
|
|
||||||
}
|
|
||||||
|
|
||||||
static void ComputeThreadJobExec__dtor(void* arg) {
|
|
||||||
ComputeThreadJobExec* self = arg;
|
|
||||||
PK_FREE(self->source);
|
|
||||||
}
|
|
||||||
|
|
||||||
typedef struct c11_ComputeThread {
|
|
||||||
int vm_index;
|
|
||||||
atomic_bool is_done;
|
|
||||||
unsigned char* last_retval_data;
|
|
||||||
int last_retval_size;
|
|
||||||
char* last_error;
|
|
||||||
|
|
||||||
c11_thrd_t thread;
|
|
||||||
void* job;
|
|
||||||
void (*job_dtor)(void*);
|
|
||||||
} c11_ComputeThread;
|
|
||||||
|
|
||||||
static void
|
|
||||||
c11_ComputeThread__reset_job(c11_ComputeThread* self, void* job, void (*job_dtor)(void*)) {
|
|
||||||
if(self->job) {
|
|
||||||
self->job_dtor(self->job);
|
|
||||||
PK_FREE(self->job);
|
|
||||||
}
|
|
||||||
self->job = job;
|
|
||||||
self->job_dtor = job_dtor;
|
|
||||||
}
|
|
||||||
|
|
||||||
static bool _pk_compute_thread_flags[16];
|
|
||||||
|
|
||||||
static void c11_ComputeThread__dtor(c11_ComputeThread* self) {
|
|
||||||
if(!self->is_done) {
|
|
||||||
c11__abort("ComputeThread(%d) is not done yet!! But the object was deleted.",
|
|
||||||
self->vm_index);
|
|
||||||
}
|
|
||||||
if(self->last_retval_data) PK_FREE(self->last_retval_data);
|
|
||||||
if(self->last_error) PK_FREE(self->last_error);
|
|
||||||
c11_ComputeThread__reset_job(self, NULL, NULL);
|
|
||||||
_pk_compute_thread_flags[self->vm_index] = false;
|
|
||||||
}
|
|
||||||
|
|
||||||
static void c11_ComputeThread__on_job_begin(c11_ComputeThread* self) {
|
|
||||||
if(self->last_retval_data) {
|
|
||||||
PK_FREE(self->last_retval_data);
|
|
||||||
self->last_retval_data = NULL;
|
|
||||||
self->last_retval_size = 0;
|
|
||||||
}
|
|
||||||
if(self->last_error) {
|
|
||||||
PK_FREE(self->last_error);
|
|
||||||
self->last_error = NULL;
|
|
||||||
}
|
|
||||||
py_switchvm(self->vm_index);
|
|
||||||
}
|
|
||||||
|
|
||||||
static bool ComputeThread__new__(int argc, py_Ref argv) {
|
|
||||||
c11_ComputeThread* self =
|
|
||||||
py_newobject(py_retval(), py_totype(argv), 0, sizeof(c11_ComputeThread));
|
|
||||||
self->vm_index = 0;
|
|
||||||
self->is_done = true;
|
|
||||||
self->last_retval_data = NULL;
|
|
||||||
self->last_retval_size = 0;
|
|
||||||
self->last_error = NULL;
|
|
||||||
self->job = NULL;
|
|
||||||
self->job_dtor = NULL;
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
static bool ComputeThread__init__(int argc, py_Ref argv) {
|
|
||||||
PY_CHECK_ARGC(2);
|
|
||||||
PY_CHECK_ARG_TYPE(1, tp_int);
|
|
||||||
c11_ComputeThread* self = py_touserdata(py_arg(0));
|
|
||||||
int index = py_toint(py_arg(1));
|
|
||||||
if(index >= 1 && index < 16) {
|
|
||||||
if(_pk_compute_thread_flags[index]) {
|
|
||||||
return ValueError("vm_index %d is already in use", index);
|
|
||||||
}
|
|
||||||
_pk_compute_thread_flags[index] = true;
|
|
||||||
self->vm_index = index;
|
|
||||||
} else {
|
|
||||||
return ValueError("vm_index %d is out of range", index);
|
|
||||||
}
|
|
||||||
py_newnone(py_retval());
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
static bool ComputeThread_is_done(int argc, py_Ref argv) {
|
|
||||||
PY_CHECK_ARGC(1);
|
|
||||||
c11_ComputeThread* self = py_touserdata(argv);
|
|
||||||
py_newbool(py_retval(), self->is_done);
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
static bool ComputeThread_wait_for_done(int argc, py_Ref argv) {
|
|
||||||
PY_CHECK_ARGC(1);
|
|
||||||
c11_ComputeThread* self = py_touserdata(argv);
|
|
||||||
while(!self->is_done)
|
|
||||||
c11_thrd_yield();
|
|
||||||
py_newnone(py_retval());
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
static bool ComputeThread_last_error(int argc, py_Ref argv) {
|
|
||||||
PY_CHECK_ARGC(1);
|
|
||||||
c11_ComputeThread* self = py_touserdata(argv);
|
|
||||||
if(!self->is_done) return OSError("thread is not done yet");
|
|
||||||
if(self->last_error) {
|
|
||||||
py_newstr(py_retval(), self->last_error);
|
|
||||||
} else {
|
|
||||||
py_newnone(py_retval());
|
|
||||||
}
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
static bool ComputeThread_last_retval(int argc, py_Ref argv) {
|
|
||||||
PY_CHECK_ARGC(1);
|
|
||||||
c11_ComputeThread* self = py_touserdata(argv);
|
|
||||||
if(!self->is_done) return OSError("thread is not done yet");
|
|
||||||
if(self->last_retval_data == NULL) return ValueError("no retval available");
|
|
||||||
return py_pickle_loads(self->last_retval_data, self->last_retval_size);
|
|
||||||
}
|
|
||||||
|
|
||||||
static c11_thrd_retval_t ComputeThreadJob_call(void* arg) {
|
|
||||||
ComputeThreadJobCall* job = arg;
|
|
||||||
c11_ComputeThread* self = job->self;
|
|
||||||
c11_ComputeThread__on_job_begin(self);
|
|
||||||
|
|
||||||
py_StackRef p0 = py_peek(0);
|
|
||||||
|
|
||||||
if(!py_pusheval(job->eval_src, NULL)) goto __ERROR;
|
|
||||||
// [callable]
|
|
||||||
if(!py_pickle_loads(job->args_data, job->args_size)) goto __ERROR;
|
|
||||||
py_push(py_retval());
|
|
||||||
// [callable, args]
|
|
||||||
if(!py_pickle_loads(job->kwargs_data, job->kwargs_size)) goto __ERROR;
|
|
||||||
py_push(py_retval());
|
|
||||||
// [callable, args, kwargs]
|
|
||||||
if(!py_smarteval("_0(*_1, **_2)", NULL, py_peek(-3), py_peek(-2), py_peek(-1))) goto __ERROR;
|
|
||||||
|
|
||||||
py_shrink(3);
|
|
||||||
if(!py_pickle_dumps(py_retval())) goto __ERROR;
|
|
||||||
int retval_size;
|
|
||||||
unsigned char* retval_data = py_tobytes(py_retval(), &retval_size);
|
|
||||||
self->last_retval_data = c11_memdup(retval_data, retval_size);
|
|
||||||
self->last_retval_size = retval_size;
|
|
||||||
self->is_done = true;
|
|
||||||
return (c11_thrd_retval_t)0;
|
|
||||||
|
|
||||||
__ERROR:
|
|
||||||
self->last_error = py_formatexc();
|
|
||||||
self->is_done = true;
|
|
||||||
py_clearexc(p0);
|
|
||||||
py_newnone(py_retval());
|
|
||||||
return (c11_thrd_retval_t)0;
|
|
||||||
}
|
|
||||||
|
|
||||||
static c11_thrd_retval_t ComputeThreadJob_exec(void* arg) {
|
|
||||||
ComputeThreadJobExec* job = arg;
|
|
||||||
c11_ComputeThread* self = job->self;
|
|
||||||
c11_ComputeThread__on_job_begin(self);
|
|
||||||
|
|
||||||
py_StackRef p0 = py_peek(0);
|
|
||||||
if(!py_exec(job->source, "<job>", job->mode, NULL)) goto __ERROR;
|
|
||||||
if(!py_pickle_dumps(py_retval())) goto __ERROR;
|
|
||||||
int retval_size;
|
|
||||||
unsigned char* retval_data = py_tobytes(py_retval(), &retval_size);
|
|
||||||
self->last_retval_data = c11_memdup(retval_data, retval_size);
|
|
||||||
self->last_retval_size = retval_size;
|
|
||||||
self->is_done = true;
|
|
||||||
return (c11_thrd_retval_t)0;
|
|
||||||
|
|
||||||
__ERROR:
|
|
||||||
self->last_error = py_formatexc();
|
|
||||||
self->is_done = true;
|
|
||||||
py_clearexc(p0);
|
|
||||||
return (c11_thrd_retval_t)0;
|
|
||||||
}
|
|
||||||
|
|
||||||
static bool ComputeThread_submit_exec(int argc, py_Ref argv) {
|
|
||||||
PY_CHECK_ARGC(2);
|
|
||||||
c11_ComputeThread* self = py_touserdata(py_arg(0));
|
|
||||||
if(!self->is_done) return OSError("thread is not done yet");
|
|
||||||
PY_CHECK_ARG_TYPE(1, tp_str);
|
|
||||||
const char* source = py_tostr(py_arg(1));
|
|
||||||
/**************************/
|
|
||||||
ComputeThreadJobExec* job = PK_MALLOC(sizeof(ComputeThreadJobExec));
|
|
||||||
job->self = self;
|
|
||||||
job->source = c11_strdup(source);
|
|
||||||
job->mode = EXEC_MODE;
|
|
||||||
c11_ComputeThread__reset_job(self, job, ComputeThreadJobExec__dtor);
|
|
||||||
/**************************/
|
|
||||||
self->is_done = false;
|
|
||||||
bool ok = c11_thrd_create(&self->thread, ComputeThreadJob_exec, job);
|
|
||||||
if(!ok) {
|
|
||||||
self->is_done = true;
|
|
||||||
return OSError("thrd_create() failed");
|
|
||||||
}
|
|
||||||
py_newnone(py_retval());
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
static bool ComputeThread_submit_eval(int argc, py_Ref argv) {
|
|
||||||
PY_CHECK_ARGC(2);
|
|
||||||
c11_ComputeThread* self = py_touserdata(py_arg(0));
|
|
||||||
if(!self->is_done) return OSError("thread is not done yet");
|
|
||||||
PY_CHECK_ARG_TYPE(1, tp_str);
|
|
||||||
const char* source = py_tostr(py_arg(1));
|
|
||||||
/**************************/
|
|
||||||
ComputeThreadJobExec* job = PK_MALLOC(sizeof(ComputeThreadJobExec));
|
|
||||||
job->self = self;
|
|
||||||
job->source = c11_strdup(source);
|
|
||||||
job->mode = EVAL_MODE;
|
|
||||||
c11_ComputeThread__reset_job(self, job, ComputeThreadJobExec__dtor);
|
|
||||||
/**************************/
|
|
||||||
self->is_done = false;
|
|
||||||
bool ok = c11_thrd_create(&self->thread, ComputeThreadJob_exec, job);
|
|
||||||
if(!ok) {
|
|
||||||
self->is_done = true;
|
|
||||||
return OSError("thrd_create() failed");
|
|
||||||
}
|
|
||||||
py_newnone(py_retval());
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
static bool ComputeThread_submit_call(int argc, py_Ref argv) {
|
|
||||||
PY_CHECK_ARGC(4);
|
|
||||||
c11_ComputeThread* self = py_touserdata(py_arg(0));
|
|
||||||
if(!self->is_done) return OSError("thread is not done yet");
|
|
||||||
PY_CHECK_ARG_TYPE(1, tp_str);
|
|
||||||
PY_CHECK_ARG_TYPE(2, tp_tuple);
|
|
||||||
PY_CHECK_ARG_TYPE(3, tp_dict);
|
|
||||||
// eval_src
|
|
||||||
const char* eval_src = py_tostr(py_arg(1));
|
|
||||||
// *args
|
|
||||||
if(!py_pickle_dumps(py_arg(2))) return false;
|
|
||||||
int args_size;
|
|
||||||
unsigned char* args_data = py_tobytes(py_retval(), &args_size);
|
|
||||||
// *kwargs
|
|
||||||
if(!py_pickle_dumps(py_arg(3))) return false;
|
|
||||||
int kwargs_size;
|
|
||||||
unsigned char* kwargs_data = py_tobytes(py_retval(), &kwargs_size);
|
|
||||||
/**************************/
|
|
||||||
ComputeThreadJobCall* job = PK_MALLOC(sizeof(ComputeThreadJobCall));
|
|
||||||
job->self = self;
|
|
||||||
job->eval_src = c11_strdup(eval_src);
|
|
||||||
job->args_data = c11_memdup(args_data, args_size);
|
|
||||||
job->args_size = args_size;
|
|
||||||
job->kwargs_data = c11_memdup(kwargs_data, kwargs_size);
|
|
||||||
job->kwargs_size = kwargs_size;
|
|
||||||
c11_ComputeThread__reset_job(self, job, ComputeThreadJobCall__dtor);
|
|
||||||
/**************************/
|
|
||||||
self->is_done = false;
|
|
||||||
bool ok = c11_thrd_create(&self->thread, ComputeThreadJob_call, job);
|
|
||||||
if(!ok) {
|
|
||||||
self->is_done = true;
|
|
||||||
return OSError("thrd_create() failed");
|
|
||||||
}
|
|
||||||
py_newnone(py_retval());
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
static bool c11_ComputeThread__exec_blocked(c11_ComputeThread* self,
|
|
||||||
const char* source,
|
|
||||||
enum py_CompileMode mode) {
|
|
||||||
if(!self->is_done) return OSError("thread is not done yet");
|
|
||||||
self->is_done = false;
|
|
||||||
char* err = NULL;
|
|
||||||
int old_vm_index = py_currentvm();
|
|
||||||
py_switchvm(self->vm_index);
|
|
||||||
py_StackRef p0 = py_peek(0);
|
|
||||||
if(!py_exec(source, "<job_blocked>", mode, NULL)) goto __ERROR;
|
|
||||||
if(!py_pickle_dumps(py_retval())) goto __ERROR;
|
|
||||||
|
|
||||||
int retval_size;
|
|
||||||
unsigned char* retval_data = py_tobytes(py_retval(), &retval_size);
|
|
||||||
py_switchvm(old_vm_index);
|
|
||||||
bool ok = py_pickle_loads(retval_data, retval_size);
|
|
||||||
self->is_done = true;
|
|
||||||
return ok;
|
|
||||||
|
|
||||||
__ERROR:
|
|
||||||
err = py_formatexc();
|
|
||||||
py_clearexc(p0);
|
|
||||||
py_switchvm(old_vm_index);
|
|
||||||
self->is_done = true;
|
|
||||||
RuntimeError("c11_ComputeThread__exec_blocked() failed:\n%s", err);
|
|
||||||
PK_FREE(err);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
static bool ComputeThread_exec(int argc, py_Ref argv) {
|
|
||||||
PY_CHECK_ARGC(2);
|
|
||||||
c11_ComputeThread* self = py_touserdata(py_arg(0));
|
|
||||||
PY_CHECK_ARG_TYPE(1, tp_str);
|
|
||||||
const char* source = py_tostr(py_arg(1));
|
|
||||||
return c11_ComputeThread__exec_blocked(self, source, EXEC_MODE);
|
|
||||||
}
|
|
||||||
|
|
||||||
static bool ComputeThread_eval(int argc, py_Ref argv) {
|
|
||||||
PY_CHECK_ARGC(2);
|
|
||||||
c11_ComputeThread* self = py_touserdata(py_arg(0));
|
|
||||||
PY_CHECK_ARG_TYPE(1, tp_str);
|
|
||||||
const char* source = py_tostr(py_arg(1));
|
|
||||||
return c11_ComputeThread__exec_blocked(self, source, EVAL_MODE);
|
|
||||||
}
|
|
||||||
|
|
||||||
static void pk_ComputeThread__register(py_Ref mod) {
|
|
||||||
py_Type type = py_newtype("ComputeThread", tp_object, mod, (py_Dtor)c11_ComputeThread__dtor);
|
|
||||||
|
|
||||||
py_bindmagic(type, __new__, ComputeThread__new__);
|
|
||||||
py_bindmagic(type, __init__, ComputeThread__init__);
|
|
||||||
py_bindproperty(type, "is_done", ComputeThread_is_done, NULL);
|
|
||||||
py_bindmethod(type, "wait_for_done", ComputeThread_wait_for_done);
|
|
||||||
py_bindmethod(type, "last_error", ComputeThread_last_error);
|
|
||||||
py_bindmethod(type, "last_retval", ComputeThread_last_retval);
|
|
||||||
|
|
||||||
py_bindmethod(type, "submit_exec", ComputeThread_submit_exec);
|
|
||||||
py_bindmethod(type, "submit_eval", ComputeThread_submit_eval);
|
|
||||||
py_bind(py_tpobject(type), "submit_call(self, eval_src, *args, **kwargs)", ComputeThread_submit_call);
|
|
||||||
|
|
||||||
py_bindmethod(type, "exec", ComputeThread_exec);
|
|
||||||
py_bindmethod(type, "eval", ComputeThread_eval);
|
|
||||||
}
|
|
||||||
|
|
||||||
void pk__add_module_pkpy() {
|
void pk__add_module_pkpy() {
|
||||||
py_Ref mod = py_newmodule("pkpy");
|
py_Ref mod = py_newmodule("pkpy");
|
||||||
|
|
||||||
@ -453,10 +99,6 @@ void pk__add_module_pkpy() {
|
|||||||
|
|
||||||
py_bindfunc(mod, "memory_usage", pkpy_memory_usage);
|
py_bindfunc(mod, "memory_usage", pkpy_memory_usage);
|
||||||
py_bindfunc(mod, "is_user_defined_type", pkpy_is_user_defined_type);
|
py_bindfunc(mod, "is_user_defined_type", pkpy_is_user_defined_type);
|
||||||
|
|
||||||
py_bindfunc(mod, "currentvm", pkpy_currentvm);
|
|
||||||
|
|
||||||
pk_ComputeThread__register(mod);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#undef DEF_TVALUE_METHODS
|
#undef DEF_TVALUE_METHODS
|
||||||
@ -1,49 +0,0 @@
|
|||||||
#include "pocketpy.h"
|
|
||||||
#include "threads.h"
|
|
||||||
#include <stdio.h>
|
|
||||||
|
|
||||||
int run_huge_job_in_vm1(void* arg) {
|
|
||||||
py_switchvm(1);
|
|
||||||
bool ok = py_exec((const char*)arg, "<job>", EXEC_MODE, NULL);
|
|
||||||
if(!ok) {
|
|
||||||
py_printexc();
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
int main() {
|
|
||||||
py_initialize();
|
|
||||||
|
|
||||||
bool ok = py_exec("print('Hello world from VM0!')", "<string1>", EXEC_MODE, NULL);
|
|
||||||
if(!ok) {
|
|
||||||
py_printexc();
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
printf("main vm index: %d\n", py_currentvm());
|
|
||||||
|
|
||||||
char* job_string =
|
|
||||||
"import time\n"
|
|
||||||
"res = 0\n"
|
|
||||||
"time.sleep(3)\n"
|
|
||||||
"res = 100\n"
|
|
||||||
"print('Huge job done!')\n"
|
|
||||||
"print('Result:', res)\n";
|
|
||||||
|
|
||||||
thrd_t thread1;
|
|
||||||
thrd_create(&thread1, run_huge_job_in_vm1, job_string);
|
|
||||||
|
|
||||||
for(int i = 0; i < 5; i++) {
|
|
||||||
thrd_sleep(&(struct timespec){.tv_sec = 1, .tv_nsec = 0}, NULL);
|
|
||||||
printf("main vm index: %d\n", py_currentvm());
|
|
||||||
}
|
|
||||||
|
|
||||||
int thrd_res;
|
|
||||||
thrd_join(thread1, &thrd_res);
|
|
||||||
printf("Thread result: %d\n", thrd_res);
|
|
||||||
|
|
||||||
py_finalize();
|
|
||||||
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
@ -1,33 +0,0 @@
|
|||||||
from pkpy import ComputeThread
|
|
||||||
import time
|
|
||||||
|
|
||||||
thread_1 = ComputeThread(1)
|
|
||||||
thread_2 = ComputeThread(2)
|
|
||||||
|
|
||||||
for t in [thread_1, thread_2]:
|
|
||||||
t.exec('''
|
|
||||||
def func(a):
|
|
||||||
from pkpy import currentvm
|
|
||||||
print("Hello from thread", currentvm(), "a =", a)
|
|
||||||
for i in range(500000):
|
|
||||||
if i % 100000 == 0:
|
|
||||||
print(i, "from thread", currentvm())
|
|
||||||
return a
|
|
||||||
|
|
||||||
x = 123
|
|
||||||
''')
|
|
||||||
assert thread_1.eval('x') == 123
|
|
||||||
|
|
||||||
# thread_1.wait_for_done()
|
|
||||||
# thread_2.wait_for_done()
|
|
||||||
|
|
||||||
thread_1.submit_call('func', [1, 2, 3])
|
|
||||||
thread_2.submit_call('func', [4, 5, 6])
|
|
||||||
|
|
||||||
while not thread_1.is_done or not thread_2.is_done:
|
|
||||||
print("Waiting for threads to finish...")
|
|
||||||
time.sleep(1)
|
|
||||||
|
|
||||||
print("Thread 1 last return value:", thread_1.last_retval())
|
|
||||||
print("Thread 2 last return value:", thread_2.last_retval())
|
|
||||||
|
|
||||||
Loading…
x
Reference in New Issue
Block a user