Merge pull request #42 from Menooker/master

add JSON serializer & deserializer

Birdee.sln

@@ -10,11 +10,6 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Birdee", "Birdee\Birdee.vcx
 		{82C8B6DF-D7E8-4934-ADAA-7FC1A88F3F46} = {82C8B6DF-D7E8-4934-ADAA-7FC1A88F3F46}
 	EndProjectSection
 EndProject
-Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "BirdeeMainTest", "BirdeeMainTest\BirdeeMainTest.vcxproj", "{82D04493-F23B-4A46-A2E0-937CE649C5D1}"
-	ProjectSection(ProjectDependencies) = postProject
-		{F07E95CE-12AE-49A2-BC62-694EE3E8B0F7} = {F07E95CE-12AE-49A2-BC62-694EE3E8B0F7}
-	EndProjectSection
-EndProject
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "BirdeeRuntime", "BirdeeRuntime\BirdeeRuntime.vcxproj", "{F07E95CE-12AE-49A2-BC62-694EE3E8B0F7}"
 EndProject
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "BirdeeCompilerCore", "BirdeeCompilerCore\BirdeeCompilerCore.vcxproj", "{793F4745-C386-4C59-A84E-00EED244B34A}"
@@ -61,18 +56,6 @@ Global
 		{B5007550-4998-4B5F-9891-4E37FCBD737B}.Release|x64.Build.0 = Release|x64
 		{B5007550-4998-4B5F-9891-4E37FCBD737B}.Release|x86.ActiveCfg = Release|Win32
 		{B5007550-4998-4B5F-9891-4E37FCBD737B}.Release|x86.Build.0 = Release|Win32
-		{82D04493-F23B-4A46-A2E0-937CE649C5D1}.Debug|x64.ActiveCfg = Debug|x64
-		{82D04493-F23B-4A46-A2E0-937CE649C5D1}.Debug|x64.Build.0 = Debug|x64
-		{82D04493-F23B-4A46-A2E0-937CE649C5D1}.Debug|x86.ActiveCfg = Debug|Win32
-		{82D04493-F23B-4A46-A2E0-937CE649C5D1}.Debug|x86.Build.0 = Debug|Win32
-		{82D04493-F23B-4A46-A2E0-937CE649C5D1}.DynLLVM|x64.ActiveCfg = DynLLVM|x64
-		{82D04493-F23B-4A46-A2E0-937CE649C5D1}.DynLLVM|x64.Build.0 = DynLLVM|x64
-		{82D04493-F23B-4A46-A2E0-937CE649C5D1}.DynLLVM|x86.ActiveCfg = DynLLVM|Win32
-		{82D04493-F23B-4A46-A2E0-937CE649C5D1}.DynLLVM|x86.Build.0 = DynLLVM|Win32
-		{82D04493-F23B-4A46-A2E0-937CE649C5D1}.Release|x64.ActiveCfg = Release|x64
-		{82D04493-F23B-4A46-A2E0-937CE649C5D1}.Release|x64.Build.0 = Release|x64
-		{82D04493-F23B-4A46-A2E0-937CE649C5D1}.Release|x86.ActiveCfg = Release|Win32
-		{82D04493-F23B-4A46-A2E0-937CE649C5D1}.Release|x86.Build.0 = Release|Win32
 		{F07E95CE-12AE-49A2-BC62-694EE3E8B0F7}.Debug|x64.ActiveCfg = Debug|x64
 		{F07E95CE-12AE-49A2-BC62-694EE3E8B0F7}.Debug|x64.Build.0 = Debug|x64
 		{F07E95CE-12AE-49A2-BC62-694EE3E8B0F7}.Debug|x86.ActiveCfg = Debug|Win32

Birdee/CodeGen.cpp

@@ -2627,8 +2627,16 @@ llvm::Value * Birdee::CallExprAST::Generate()
 	if (memberexpr) //if it is a member expr AST of a member function
 	{
 		memberexpr->GenerateObj();
-		if(outfunc)
+		if (outfunc)
+		{
+			if (memberexpr->Obj && !memberexpr->llvm_obj)
+			{
+				//if there is a object reference and we only have a RValue of struct
+				memberexpr->llvm_obj = builder.CreateAlloca(memberexpr->Obj->resolved_type.class_ast->llvm_type); //alloca temp memory for the value
+				builder.CreateStore(memberexpr->Obj->Generate(), memberexpr->llvm_obj); //store the obj to the alloca
+			}
 			func = outfunc->GetLLVMFunc();
+		}
 		else
 			func = memberexpr->GenerateFunction();
 		obj = memberexpr->llvm_obj;

Birdee/MetadataDeserializer.cpp

@@ -19,6 +19,8 @@ using namespace Birdee;
 static std::vector<ClassAST*> idx_to_class;
 static std::vector<PrototypeAST*> idx_to_proto;
 static std::vector<ClassAST*> orphan_idx_to_class;
+//the orphan classes that needs to call PreGenerate on
+static std::vector<ClassAST*> orphan_class_to_generate;
 static std::vector<ClassAST*> to_run_phase0;
 static string current_package_name;
 static int current_module_idx;
@@ -488,6 +490,8 @@ void PreBuildOneOrphanClassFromJson(const json& cls, int idx,
 	string name = json_cls["name"].get<string>();
 	auto orphan = cu.orphan_class.find(name);
 	ClassAST* classdef;
+	//whether we need to call PreGenerate on this class?
+	bool needs_generate = true;
 	if (orphan != cu.orphan_class.end())
 	{//if already imported in orphan classes
 		classdef = orphan->second.get();
@@ -501,6 +505,7 @@ void PreBuildOneOrphanClassFromJson(const json& cls, int idx,
 		auto clsdef_itr = src_mod->classmap.find(cls_simple_name);
 		BirdeeAssert(clsdef_itr != src_mod->classmap.end(), "Cannot find the class in pre-imported module");
 		classdef = clsdef_itr->second.first.get();
+		needs_generate = false;
 	}
 	else
 	{
@@ -547,6 +552,10 @@ void PreBuildOneOrphanClassFromJson(const json& cls, int idx,
 					classdef->template_source_class = src;
 					src->template_param->AddImpl(vec, std::move(newclass));
 				}
+				else
+				{
+					needs_generate = false;
+				}
 			}
 			else
 			{//if the template itself is not imported, make it an orphan
@@ -564,6 +573,7 @@ void PreBuildOneOrphanClassFromJson(const json& cls, int idx,
 				auto itr = node->mod->classmap.find(name.substr(idx + 1));
 				BirdeeAssert(itr != node->mod->classmap.end(), "Module imported, but cannot find the class");
 				classdef = itr->second.first.get();
+				needs_generate = false;
 			}
 			else
 			{
@@ -575,6 +585,8 @@ void PreBuildOneOrphanClassFromJson(const json& cls, int idx,
 
 	}
 	orphan_idx_to_class[idx] = classdef;
+	if (needs_generate)
+		orphan_class_to_generate.push_back(classdef);
 }
 
 void PreBuildOrphanClassFromJson(const json& cls, 
@@ -666,14 +678,14 @@ void BuildClassFromJson(const json& cls, ImportedModule& mod)
 		//fix-me: check if the imported type is the same as the existing type
 
 
-		auto srcitr = json_cls.find("source");
+		auto srcitr = json_cls.find("template_source");
 		if (srcitr != json_cls.end()) //if it's a class template instance
 		{
 			int src_idx = srcitr->get<int>();
 			BirdeeAssert(src_idx >= 0 && src_idx < idx_to_class.size(), "Template source index out of index");
 			ClassAST* src = idx_to_class[src_idx];
 			BirdeeAssert(src->isTemplate(), "The source must be a class template");
-			auto targs = BuildTemplateArgsFromJson(json_cls["arguments"]);
+			auto targs = BuildTemplateArgsFromJson(json_cls["template_arguments"]);
 			auto name_cls_itr = mod.classmap.find((*itr)->name);
 			assert(name_cls_itr != mod.classmap.end());
 			auto oldcls = src->template_param->Get(*targs);
@@ -814,6 +826,7 @@ void ImportedModule::Init(const vector<string>& package, const string& module_na
 	idx_to_class.clear();
 	to_run_phase0.clear();
 	orphan_idx_to_class.clear();
+	orphan_class_to_generate.clear();
 	source_paths.push_back(path);
 	BirdeeAssert(json["Type"].get<string>() == "Birdee Module Metadata", "Bad file type");
 	BirdeeAssert(json["Version"].get<double>() <= META_DATA_VERSION, "Unsupported version");
@@ -852,7 +865,7 @@ void ImportedModule::Init(const vector<string>& package, const string& module_na
 		cls.second.first->PreGenerateFuncs();
 		//cls.second->Generate();
 	}
-	for (auto cls : orphan_idx_to_class)
+	for (auto cls : orphan_class_to_generate)
 	{
 		cls->PreGenerate();
 		cls->PreGenerateFuncs();

Birdee/MetadataSerializer.cpp

@@ -105,13 +105,13 @@ int ConvertClassToIndex(ClassAST* class_ast)
 				defined_classes->push_back(json());
 				(*defined_classes)[retidx] = BuildSingleClassJson(*class_ast, false);
 				auto& the_class = (*defined_classes)[retidx];
-				the_class["source"] = ConvertClassToIndex(class_ast->template_source_class);
+				the_class["template_source"] = ConvertClassToIndex(class_ast->template_source_class);
 				auto args = json::array();
 				for (auto& arg : *class_ast->template_instance_args)
 				{
 					args.push_back(BuildTemplateArgumentJson(arg));
 				}
-				the_class["arguments"] = args;
+				the_class["template_arguments"] = args;
 				//class_ast->template_instance_args
 				return retidx;
 			}

Birdee/Parser.cpp

@@ -1243,7 +1243,7 @@ std::unique_ptr<FunctionAST> ParseDeclareFunction(ClassAST* cls)
 //is_field: true for FieldDef, false for MemberFunctionDef
 //index: the index within fields/funcdef
 static const string INTERNAL_ANNO_VIRTUAL = "virtual";
-// static const string INTERNAL_ANNO_PURE_VIRTUAL = "pure_virtual";
+static const string INTERNAL_ANNO_STACK_CAPTURE = "stack_capture";
 static unordered_map<string, std::function<void(ClassAST* cls, bool is_field, int index)>> interal_class_annontation_map = {
 	{INTERNAL_ANNO_VIRTUAL, [](ClassAST* cls, bool is_field, int index) {
 		CompileAssert(!is_field,"The \'virtual\' annotation can only be applied on member functions");
@@ -1252,6 +1252,10 @@ static unordered_map<string, std::function<void(ClassAST* cls, bool is_field, in
 		cls->needs_rtti = true;
 		//it is set to VIRT_UNRESOLVED if is marked virtual but unresolved before Phase0
 		cls->funcs[index].virtual_idx = MemberFunctionDef::VIRT_UNRESOLVED;
+	}},
+	{INTERNAL_ANNO_STACK_CAPTURE, [](ClassAST* cls, bool is_field, int index) {
+		CompileAssert(!is_field,"The \'stack_capture\' annotation can only be applied on member functions");
+		cls->funcs[index].decl->capture_on_stack = true;
 	}}
 	/*
 	{INTERNAL_ANNO_PURE_VIRTUAL, [](ClassAST* cls, bool is_field, int index) {

Birdee/Preprocessing.cpp

@@ -1137,9 +1137,7 @@ namespace Birdee
 				[&this_template_args](IdentifierExprAST* ex) {
 				ImportTree* import_tree = nullptr;
 				auto ret = scope_mgr.ResolveNameNoThrow(ex->Name, ex->Pos, import_tree);
-				if (import_tree)
-					assert(0 && "Not implemented");
-				if (ret)
+				if (ret && !import_tree)
 				{
 					this_template_args.push_back(TemplateArgument(std::move(ret)));
 				}
@@ -1589,16 +1587,16 @@ namespace Birdee
 	{
 		preprocessing_state.current_phase = 1;
 		auto len = preprocessing_state.class_templ_inst_rollback.size();
-		for (size_t i = 0; i < len; i++)
-		{
-			// in phase0, the phase1 of template instances are not called. Call phase1 here with proper environment
-			Phase1OnlyForTemplateInstance(*preprocessing_state.class_templ_inst_rollback.at(i));
-		}
 		//scope_mgr.PushBasicBlock();
 		for (auto& stmt : toplevel)
 		{
 			stmt->Phase1();
 		}
+		for (size_t i = 0; i < len; i++)
+		{
+			// in phase0, the phase1 of template instances are not called. Call phase1 here with proper environment
+			Phase1OnlyForTemplateInstance(*preprocessing_state.class_templ_inst_rollback.at(i));
+		}
 		//scope_mgr.PopBasicBlock();
 	}
 

Birdee/PythonBinding2.cpp

@@ -160,6 +160,10 @@ BIRDEE_BINDING_API string Birdee_RunScriptForString(const string& str, const Sou
 
 BIRDEE_BINDING_API void Birdee_ScriptAST_Phase1(ScriptAST* ths, void* globals, void* locals)
 {
+	auto old_type = outtype;
+	auto old_scriptast = cur_script_ast;
+	auto old_expr = std::move(outexpr);
+
 	cur_script_ast = ths;
 	auto& env=InitPython();
 	try
@@ -198,9 +202,9 @@ BIRDEE_BINDING_API void Birdee_ScriptAST_Phase1(ScriptAST* ths, void* globals, v
 	{
 		ths->type_data = outtype;
 	}
-	outexpr.clear();
-	outtype = ResolvedType();
-	cur_script_ast = nullptr;
+	outexpr = std::move(old_expr);
+	outtype = old_type;
+	cur_script_ast = old_scriptast;
 }
 
 static UniquePtrStatementAST CompileExpr(char* cmd) {
@@ -318,6 +322,11 @@ static auto NewNumberExpr(Token tok, py::object& obj) {
 
 BIRDEE_BINDING_API void Birdee_ScriptType_Resolve(ResolvedType* out, ScriptType* ths,SourcePos pos,void* globals, void* locals)
 {
+	auto old_type = outtype;
+	auto old_scriptast = cur_script_ast;
+	auto old_expr = std::move(outexpr);
+
+	cur_script_ast = nullptr;
 	auto& env = InitPython();
 	try
 	{
@@ -333,8 +342,10 @@ BIRDEE_BINDING_API void Birdee_ScriptType_Resolve(ResolvedType* out, ScriptType*
 		throw CompileError(pos, "The returned type is invalid");
 	}
 	*out = outtype;
-	outtype.type = tok_error;
-	outexpr.clear();
+
+	cur_script_ast = old_scriptast;
+	outtype = old_type;
+	outexpr = std::move(old_expr);
 }
 
 BIRDEE_BINDING_API void Birdee_RunAnnotationsOn(std::vector<std::string>& anno,StatementAST* impl,SourcePos pos, void* globals)

BirdeeHome/pylib/bdutils.py

@@ -111,3 +111,27 @@ def func_templ_expr_at(idx, thefunc = None):
 	if not thefunc:
 		thefunc = get_cur_func()
 	set_ast(get_func_expr_templ_at(idx, thefunc))
+
+
+def _foreach_field_impl(T: ClassAST, callback):
+	fld=[]
+	def get_all_fields(cur: ClassAST):
+		if cur.parent_class:
+			get_all_fields(cur.parent_class)
+		for f in cur.fields:
+			fld.append(f)
+	get_all_fields(T)
+	length=len(fld)
+	for idx, field in enumerate(fld):
+		callback(idx, length, field)
+
+'''
+T should be a resolved type/ClassAST.
+callback should be (idx, length, field)
+'''
+def foreach_field(T, callback):
+	if not isinstance(T, ClassAST):
+		if not is_a_class(T) and not is_a_struct(T):
+			raise RuntimeError("T = {} should be a class or struct".format(T))
+		T=T.get_detail()
+	_foreach_field_impl(T, callback)

BirdeeHome/pylib/traits.py

@@ -116,6 +116,13 @@ def is_expr_templ_arg(args,idx):
 	return (args.kind==TemplateArgument.TemplateArgumentType.TEMPLATE_ARG_EXPR , 
 		lambda:"The {}-th template argument is expected to be an expression".format(idx))
 
+def get_array_element_type(arrty):
+	require_(arrty.index_level>0)
+	Ele = ResolvedType()
+	Ele.set_detail(arrty.base, arrty.get_detail())
+	Ele.index_level = arrty.index_level - 1
+	return Ele
+
 '''
 Check if "type" is a prototype. Throw if not.
  - type: The ResolvedType to check

BirdeeHome/src/Makefile

@@ -27,7 +27,7 @@ ${BLIB_DIR}/birdee.o: birdee.txt ${BLIB_DIR} ${BIN}
 libs: ${BLIB_DIR}/birdee.o
 	python3 $(PARENT_DIR)/pylib/bbuild.py -i . -o ${BLIB_DIR} \
 		variant list hash tuple fmt vector queue stack unsafe concurrent.threading rtti system.io.stdio typedptr reflection concurrent.sync system.io.net \
-		functional.lazy concurrent.threadpool system.time extensions.string system.io.stream
+		functional.lazy concurrent.threadpool system.time extensions.string system.io.stream serialization.json.serializer serialization.json.deserializer
 
 clean:
 	rm -rf ${BLIB_DIR}

BirdeeHome/src/any.bdm

@@ -50,6 +50,9 @@ else:
 @}	
 	end
 
+	public func get_type() as type_info => typeof(v)
+	public func is_null() as boolean => v===null
+
 	public func is_type[T]() as boolean
 {@
 T = get_func_type_templ_at(0)

BirdeeHome/src/extensions/string.bdm

@@ -296,4 +296,27 @@ end
 		return false
 	end
 	return v.view(v.length()-substr.length(), substr.length()) == substr
+end
+
+@extension function string_replace(v as string, substr as string, repl as string) as string
+	dim cur as int = 0
+	dim ret = new vector[byte](0)
+	ret.reserve(v.length())
+	while cur < v.length()
+		dim nxt = v.find(substr, cur)
+		if nxt==-1 then
+			ret.push_chunk(v.get_bytes(), cur, v.length() - cur)
+			break
+		end
+		ret.push_chunk(v.get_bytes(), cur, nxt - cur)
+		ret.push_chunk(repl.get_bytes(), 0, repl.length())
+		cur = nxt + substr.length()
+	end
+	return new string:copy_bytes(ret.buffer(), 0, ret.size())
+end
+
+function char2str(c as byte) as string
+	dim t = new byte*1
+	t[0]=c
+	return new string:copy_bytes(t, 0, 1)
 end

BirdeeHome/src/functional/closures.bdm

@@ -1,5 +1,7 @@
 package functional
 
+import unsafe
+
 struct closure_unpacked
 	public funcptr as pointer
 	public data as pointer

BirdeeHome/src/reflection.bdm

@@ -55,6 +55,21 @@ function get_class_field_ptr[fname as string,T](obj as T) as pointer
     return get_field_ptr[fname, T](pointerof(obj))
 end
 
+function get_class_field[fname as string,T](obj as T) as {@
+T=get_func_type_templ_at(1)
+if not is_a_class(T) or not is_a_struct(T):
+    raise RuntimeError("T = {} should be a class or struct".format(T))
+T=T.get_detail()
+fname=get_func_expr_templ_at(0).value
+fidx,fld = T.find_field(fname)
+if not fld:
+    raise RuntimeError("The field {} is not in the class {}".format(fname, T))
+TY = fld.decl.resolved_type
+set_type(TY)
+@}
+    return unsafe.ptr_load[{@set_type(TY)@}](get_field_ptr[fname, T](pointerof(obj)))
+end
+
 function register[T]()
     {@
 T = get_func_type_templ_at(0).get_detail()