From cb9e43cb6297748b1a5992b0e9f2d9fec81acad9 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Tue, 24 Feb 2026 10:34:45 +0530 Subject: [PATCH 1/6] fix(parser): add Python-compatible literal parser for C++ concore nodes - Replace stod-only parser with recursive descent parser in concore_base.hpp - Introduce ConcoreValue variant type supporting numbers, booleans, strings, nested arrays, and tuples (matching Python ast.literal_eval output) - Add parse_literal() and flatten_numeric() APIs to concore.hpp - Maintain full backward compatibility for flat numeric payloads - Add TestLiteralEvalCpp.cpp with 79 tests covering all payload types, error cases, and cross-language round-trip scenarios - Document wire format in README.md - Prevents silent cross-language data loss Fixes #389 --- README.md | 10 ++ TestLiteralEvalCpp.cpp | 307 +++++++++++++++++++++++++++++++++++++++++ concore.hpp | 21 +++ concore_base.hpp | 247 ++++++++++++++++++++++++++++++++- 4 files changed, 580 insertions(+), 5 deletions(-) create mode 100644 TestLiteralEvalCpp.cpp diff --git a/README.md b/README.md index a44827d..60c94e7 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,16 @@ The CONTROL-CORE framework consists of the below projects. _concore_ enables composing studies from programs developed in different languages. Currently supported languages are, Python, Matlab/Octave, Verilog, and C++. The studies are designed through the visual _concore_ Editor (DHGWorkflow) and interpreted into _concore_ through its parser. Neural control systems consist of loops (dicycles). Therefore, they cannot be represented by classic workflow standards (such as CWL or WDL). Therefore, _concore_ addresses a significant research gap to model closed-loop neuromodulation control systems. The _concore_ protocol shares data between the programs through file sharing, with no centralized entity (a broker or an orchestrator) to arbitrate communications between the programs. (In the distributed executions, the CONTROL-CORE Mediator enables connecting the disjoint pieces of the study through REST APIs). +## Wire Format + +Concore payloads follow Python literal syntax compatible with `ast.literal_eval()`. All language implementations (Python, C++, Java, MATLAB) parse this shared format. Supported value types include: + +* **Numbers** — integers and floats, including scientific notation (e.g., `1e3`, `-2.5`) +* **Booleans** — `True` / `False` (converted to `1.0` / `0.0` in numeric contexts) +* **Strings** — single- or double-quoted (e.g., `"start"`, `'label'`) +* **Nested arrays** — `[1, [2, 3]]` +* **Tuples** — `(1.0, 2.0)` (treated identically to arrays) + # Installation and Getting Started Guide diff --git a/TestLiteralEvalCpp.cpp b/TestLiteralEvalCpp.cpp new file mode 100644 index 0000000..5746837 --- /dev/null +++ b/TestLiteralEvalCpp.cpp @@ -0,0 +1,307 @@ +/** + * TestLiteralEvalCpp.cpp + * + * Test suite for the C++ Python-literal-compatible parser in concore_base.hpp. + * Validates Issue #389 fix: C++ parser must accept all valid concore payloads + * that Python's ast.literal_eval() accepts. + * + * Compile: g++ -std=c++11 -o TestLiteralEvalCpp TestLiteralEvalCpp.cpp + * Run: ./TestLiteralEvalCpp (Linux/macOS) + * TestLiteralEvalCpp.exe (Windows) + */ + +#include +#include +#include +#include +#include +#include + +#include "concore_base.hpp" + +using namespace concore_base; + +static int passed = 0; +static int failed = 0; + +// ------------- helpers ------------------------------------------------- + +static void check(const std::string& testName, bool condition) { + if (condition) { + std::cout << "PASS: " << testName << std::endl; + ++passed; + } else { + std::cout << "FAIL: " << testName << std::endl; + ++failed; + } +} + +static bool approx(double a, double b, double eps = 1e-9) { + return std::fabs(a - b) < eps; +} + +// ------------- backward-compatibility tests ---------------------------- + +static void test_flat_numeric_list() { + std::vector v = parselist_double("[10.0, 0.5, 2.3]"); + check("flat_numeric size==3", v.size() == 3); + check("flat_numeric[0]==10.0", approx(v[0], 10.0)); + check("flat_numeric[1]==0.5", approx(v[1], 0.5)); + check("flat_numeric[2]==2.3", approx(v[2], 2.3)); +} + +static void test_empty_list() { + std::vector v = parselist_double("[]"); + check("empty_list size==0", v.size() == 0); +} + +static void test_single_element() { + std::vector v = parselist_double("[42.0]"); + check("single_element size==1", v.size() == 1); + check("single_element[0]==42", approx(v[0], 42.0)); +} + +static void test_negative_numbers() { + std::vector v = parselist_double("[-1.5, -3.0, 2.0]"); + check("negative size==3", v.size() == 3); + check("negative[0]==-1.5", approx(v[0], -1.5)); + check("negative[1]==-3.0", approx(v[1], -3.0)); +} + +static void test_scientific_notation() { + std::vector v = parselist_double("[1e3, 2.5E-2, -1.0e+1]"); + check("sci size==3", v.size() == 3); + check("sci[0]==1000", approx(v[0], 1000.0)); + check("sci[1]==0.025", approx(v[1], 0.025)); + check("sci[2]==-10", approx(v[2], -10.0)); +} + +static void test_integer_values() { + std::vector v = parselist_double("[1, 2, 3]"); + check("int size==3", v.size() == 3); + check("int[0]==1", approx(v[0], 1.0)); + check("int[2]==3", approx(v[2], 3.0)); +} + +// ------------- mixed-type payload tests (Issue #389 core) -------------- + +static void test_string_element() { + // [10.0, "start", 0.5] – string should be skipped in numeric flatten + std::vector v = parselist_double("[10.0, \"start\", 0.5]"); + check("string_elem size==2", v.size() == 2); + check("string_elem[0]==10.0", approx(v[0], 10.0)); + check("string_elem[1]==0.5", approx(v[1], 0.5)); +} + +static void test_boolean_element() { + // [10.0, True, 0.5] + std::vector v = parselist_double("[10.0, True, 0.5]"); + check("bool_elem size==3", v.size() == 3); + check("bool_elem[0]==10.0", approx(v[0], 10.0)); + check("bool_elem[1]==1.0 (True)", approx(v[1], 1.0)); + check("bool_elem[2]==0.5", approx(v[2], 0.5)); +} + +static void test_bool_false() { + std::vector v = parselist_double("[False, 5.0]"); + check("bool_false size==2", v.size() == 2); + check("bool_false[0]==0.0", approx(v[0], 0.0)); +} + +static void test_nested_list() { + // [10.0, [0.5, 0.3], 0.1] – nested list flattened to [10.0, 0.5, 0.3, 0.1] + std::vector v = parselist_double("[10.0, [0.5, 0.3], 0.1]"); + check("nested size==4", v.size() == 4); + check("nested[0]==10.0", approx(v[0], 10.0)); + check("nested[1]==0.5", approx(v[1], 0.5)); + check("nested[2]==0.3", approx(v[2], 0.3)); + check("nested[3]==0.1", approx(v[3], 0.1)); +} + +static void test_tuple_payload() { + // (10.0, 0.3) – tuple treated as array + std::vector v = parselist_double("(10.0, 0.3)"); + check("tuple size==2", v.size() == 2); + check("tuple[0]==10.0", approx(v[0], 10.0)); + check("tuple[1]==0.3", approx(v[1], 0.3)); +} + +static void test_nested_tuple() { + // [10.0, (0.5, 0.3)] + std::vector v = parselist_double("[10.0, (0.5, 0.3)]"); + check("nested_tuple size==3", v.size() == 3); + check("nested_tuple[0]==10.0", approx(v[0], 10.0)); + check("nested_tuple[1]==0.5", approx(v[1], 0.5)); + check("nested_tuple[2]==0.3", approx(v[2], 0.3)); +} + +static void test_mixed_types() { + // [10.0, "label", True, [1, 2], (3,), False, "end"] + std::vector v = parselist_double("[10.0, \"label\", True, [1, 2], (3,), False, \"end\"]"); + // numeric values: 10.0, 1.0(True), 1, 2, 3, 0.0(False) = 6 values + check("mixed size==6", v.size() == 6); + check("mixed[0]==10.0", approx(v[0], 10.0)); + check("mixed[1]==1.0", approx(v[1], 1.0)); // True + check("mixed[2]==1.0", approx(v[2], 1.0)); // nested [1,...] + check("mixed[3]==2.0", approx(v[3], 2.0)); // nested [...,2] + check("mixed[4]==3.0", approx(v[4], 3.0)); // tuple (3,) + check("mixed[5]==0.0", approx(v[5], 0.0)); // False +} + +// ------------- full ConcoreValue parse tests --------------------------- + +static void test_parse_literal_string() { + ConcoreValue v = parse_literal("[10.0, \"start\", 0.5]"); + check("literal_string is ARRAY", v.type == ConcoreValueType::ARRAY); + check("literal_string len==3", v.array.size() == 3); + check("literal_string[0] NUMBER", v.array[0].type == ConcoreValueType::NUMBER); + check("literal_string[1] STRING", v.array[1].type == ConcoreValueType::STRING); + check("literal_string[1]==\"start\"", v.array[1].str == "start"); + check("literal_string[2] NUMBER", v.array[2].type == ConcoreValueType::NUMBER); +} + +static void test_parse_literal_bool() { + ConcoreValue v = parse_literal("[True, False]"); + check("literal_bool is ARRAY", v.type == ConcoreValueType::ARRAY); + check("literal_bool[0] BOOL", v.array[0].type == ConcoreValueType::BOOL); + check("literal_bool[0]==true", v.array[0].boolean == true); + check("literal_bool[1]==false", v.array[1].boolean == false); +} + +static void test_parse_literal_nested() { + ConcoreValue v = parse_literal("[1, [2, [3]]]"); + check("literal_nested outer ARRAY", v.type == ConcoreValueType::ARRAY); + check("literal_nested[1] ARRAY", v.array[1].type == ConcoreValueType::ARRAY); + check("literal_nested[1][1] ARRAY", v.array[1].array[1].type == ConcoreValueType::ARRAY); + check("literal_nested[1][1][0]==3", approx(v.array[1].array[1].array[0].number, 3.0)); +} + +static void test_parse_single_quoted_string() { + ConcoreValue v = parse_literal("['hello']"); + check("single_quote ARRAY", v.type == ConcoreValueType::ARRAY); + check("single_quote[0] STRING", v.array[0].type == ConcoreValueType::STRING); + check("single_quote[0]=='hello'", v.array[0].str == "hello"); +} + +static void test_parse_escape_sequences() { + ConcoreValue v = parse_literal("[\"line\\none\"]"); + check("escape STRING", v.array[0].type == ConcoreValueType::STRING); + check("escape has newline", v.array[0].str == "line\none"); +} + +static void test_parse_none() { + ConcoreValue v = parse_literal("[None, 1]"); + check("none[0] STRING", v.array[0].type == ConcoreValueType::STRING); + check("none[0]==\"None\"", v.array[0].str == "None"); +} + +static void test_trailing_comma() { + // Python allows trailing comma: [1, 2,] + std::vector v = parselist_double("[1, 2,]"); + check("trailing_comma size==2", v.size() == 2); + check("trailing_comma[1]==2", approx(v[1], 2.0)); +} + +// ------------- error / failure case tests ------------------------------ + +static void test_malformed_bracket() { + bool caught = false; + try { + parse_literal("[1, 2"); + } catch (const std::runtime_error&) { + caught = true; + } + check("malformed_bracket throws", caught); +} + +static void test_malformed_string() { + bool caught = false; + try { + parse_literal("[\"unterminated]"); + } catch (const std::runtime_error&) { + caught = true; + } + check("malformed_string throws", caught); +} + +static void test_unsupported_object() { + bool caught = false; + try { + parse_literal("{1: 2}"); + } catch (const std::runtime_error&) { + caught = true; + } + check("unsupported_object throws", caught); +} + +static void test_empty_string_input() { + std::vector v = parselist_double(""); + check("empty_input size==0", v.size() == 0); +} + +// ------------- cross-language round-trip tests ------------------------- + +static void test_python_write_cpp_read_flat() { + // Simulate Python write: "[5.0, 1.0, 2.0]" + std::vector v = parselist_double("[5.0, 1.0, 2.0]"); + check("py2cpp_flat size==3", v.size() == 3); + check("py2cpp_flat[0]==5.0", approx(v[0], 5.0)); +} + +static void test_python_write_cpp_read_mixed() { + // Simulate Python write: "[5.0, 'sensor_a', True, [0.1, 0.2]]" + std::vector v = parselist_double("[5.0, 'sensor_a', True, [0.1, 0.2]]"); + // numeric: 5.0, 1.0(True), 0.1, 0.2 = 4 + check("py2cpp_mixed size==4", v.size() == 4); + check("py2cpp_mixed[0]==5.0", approx(v[0], 5.0)); + check("py2cpp_mixed[1]==1.0", approx(v[1], 1.0)); + check("py2cpp_mixed[2]==0.1", approx(v[2], 0.1)); + check("py2cpp_mixed[3]==0.2", approx(v[3], 0.2)); +} + +// ------------- main ---------------------------------------------------- + +int main() { + std::cout << "===== C++ Literal Parser Tests (Issue #389) =====\n\n"; + + // Backward compatibility + test_flat_numeric_list(); + test_empty_list(); + test_single_element(); + test_negative_numbers(); + test_scientific_notation(); + test_integer_values(); + + // Mixed-type payloads (core of Issue #389) + test_string_element(); + test_boolean_element(); + test_bool_false(); + test_nested_list(); + test_tuple_payload(); + test_nested_tuple(); + test_mixed_types(); + + // Full ConcoreValue structure tests + test_parse_literal_string(); + test_parse_literal_bool(); + test_parse_literal_nested(); + test_parse_single_quoted_string(); + test_parse_escape_sequences(); + test_parse_none(); + test_trailing_comma(); + + // Error / failure cases + test_malformed_bracket(); + test_malformed_string(); + test_unsupported_object(); + test_empty_string_input(); + + // Cross-language round-trip + test_python_write_cpp_read_flat(); + test_python_write_cpp_read_mixed(); + + std::cout << "\n=== Results: " << passed << " passed, " << failed + << " failed out of " << (passed + failed) << " tests ===\n"; + + return (failed > 0) ? 1 : 0; +} diff --git a/concore.hpp b/concore.hpp index 3e666b3..5016a18 100644 --- a/concore.hpp +++ b/concore.hpp @@ -337,6 +337,27 @@ class Concore{ return concore_base::parselist_double(f); } + /** + * @brief Parses a Python-literal payload into a structured ConcoreValue. + * Supports numbers, booleans, strings, nested arrays, and tuples. + * Use this when you need the full parsed structure, not just doubles. + * @param f The input string to parse. + * @return A ConcoreValue representing the parsed literal. + * @throws std::runtime_error on malformed input. + */ + concore_base::ConcoreValue parse_literal(string f){ + return concore_base::parse_literal(f); + } + + /** + * @brief Recursively extracts all numeric values from a ConcoreValue. + * @param v The ConcoreValue to flatten. + * @return A flat vector of doubles. + */ + vector flatten_numeric(const concore_base::ConcoreValue& v){ + return concore_base::flatten_numeric(v); + } + /** * @brief deviate the read to either the SM (Shared Memory) or FM (File Method) communication protocol based on iport and oport. * @param port The port number. diff --git a/concore_base.hpp b/concore_base.hpp index 9018f61..d03a0e0 100644 --- a/concore_base.hpp +++ b/concore_base.hpp @@ -84,14 +84,251 @@ inline std::vector parselist(const std::string& str) { /** * Parses a double-valued list like "[0.0, 1.5, 2.3]" into a vector. * Used by concore.hpp's read/write which work with numeric data. + * Now delegates to the full literal parser to handle mixed-type payloads + * (strings, booleans, nested lists, tuples) without crashing. + * See Issue #389. */ +inline std::vector parselist_double(const std::string& str); // forward decl; defined after ConcoreValue + +// =================================================================== +// Python-Literal-Compatible Value Type and Parser (Issue #389) +// =================================================================== + +/** + * Tag for ConcoreValue discriminated union. + */ +enum class ConcoreValueType { NUMBER, BOOL, STRING, ARRAY }; + +/** + * A recursive value type that mirrors Python's ast.literal_eval output. + * Supported: numbers, booleans, strings, and nested arrays / tuples. + */ +struct ConcoreValue { + ConcoreValueType type; + double number; + bool boolean; + std::string str; + std::vector array; + + ConcoreValue() : type(ConcoreValueType::NUMBER), number(0.0), boolean(false) {} + + static ConcoreValue make_number(double v) { + ConcoreValue cv; + cv.type = ConcoreValueType::NUMBER; + cv.number = v; + return cv; + } + static ConcoreValue make_bool(bool v) { + ConcoreValue cv; + cv.type = ConcoreValueType::BOOL; + cv.boolean = v; + cv.number = v ? 1.0 : 0.0; // Python: True == 1, False == 0 + return cv; + } + static ConcoreValue make_string(const std::string& v) { + ConcoreValue cv; + cv.type = ConcoreValueType::STRING; + cv.str = v; + return cv; + } + static ConcoreValue make_array(const std::vector& v) { + ConcoreValue cv; + cv.type = ConcoreValueType::ARRAY; + cv.array = v; + return cv; + } +}; + +// --------------- internal helpers (anonymous-namespace-like) -------- + +inline void skip_ws(const std::string& s, size_t& pos) { + while (pos < s.size() && std::isspace(static_cast(s[pos]))) + ++pos; +} + +inline ConcoreValue parse_literal_value(const std::string& s, size_t& pos); + +inline ConcoreValue parse_literal_string(const std::string& s, size_t& pos) { + char quote = s[pos]; // ' or " + ++pos; + std::string result; + while (pos < s.size() && s[pos] != quote) { + if (s[pos] == '\\' && pos + 1 < s.size()) { + ++pos; + switch (s[pos]) { + case 'n': result += '\n'; break; + case 't': result += '\t'; break; + case '\\': result += '\\'; break; + case '\'': result += '\''; break; + case '"': result += '"'; break; + default: result += '\\'; result += s[pos]; break; + } + } else { + result += s[pos]; + } + ++pos; + } + if (pos >= s.size()) + throw std::runtime_error("Invalid concore payload: unterminated string"); + ++pos; // skip closing quote + return ConcoreValue::make_string(result); +} + +inline ConcoreValue parse_literal_array(const std::string& s, size_t& pos) { + char open = s[pos]; + char close = (open == '[') ? ']' : ')'; + ++pos; + std::vector elements; + skip_ws(s, pos); + if (pos < s.size() && s[pos] == close) { ++pos; return ConcoreValue::make_array(elements); } + while (pos < s.size()) { + elements.push_back(parse_literal_value(s, pos)); + skip_ws(s, pos); + if (pos < s.size() && s[pos] == ',') { ++pos; skip_ws(s, pos); } + if (pos < s.size() && s[pos] == close) { ++pos; return ConcoreValue::make_array(elements); } + } + throw std::runtime_error("Invalid concore payload: unterminated array/tuple"); +} + +/** + * Recursive descent parser entry for a single Python literal value. + * Advances `pos` past the consumed token. + */ +inline ConcoreValue parse_literal_value(const std::string& s, size_t& pos) { + skip_ws(s, pos); + if (pos >= s.size()) + throw std::runtime_error("Invalid concore payload: unexpected end of input"); + + char c = s[pos]; + + // Array / Tuple + if (c == '[' || c == '(') + return parse_literal_array(s, pos); + + // String + if (c == '\'' || c == '"') + return parse_literal_string(s, pos); + + // Boolean True + if (s.compare(pos, 4, "True") == 0 && + (pos + 4 >= s.size() || !std::isalnum(static_cast(s[pos + 4])))) { + pos += 4; + return ConcoreValue::make_bool(true); + } + // Boolean False + if (s.compare(pos, 5, "False") == 0 && + (pos + 5 >= s.size() || !std::isalnum(static_cast(s[pos + 5])))) { + pos += 5; + return ConcoreValue::make_bool(false); + } + // None → treat as string "None" (no numeric equivalent) + if (s.compare(pos, 4, "None") == 0 && + (pos + 4 >= s.size() || !std::isalnum(static_cast(s[pos + 4])))) { + pos += 4; + return ConcoreValue::make_string("None"); + } + + // Number (int, float, negative, scientific notation) + { + size_t start = pos; + if (pos < s.size() && (s[pos] == '+' || s[pos] == '-')) ++pos; + bool has_digits = false; + while (pos < s.size() && std::isdigit(static_cast(s[pos]))) { + ++pos; has_digits = true; + } + if (pos < s.size() && s[pos] == '.') { + ++pos; + while (pos < s.size() && std::isdigit(static_cast(s[pos]))) { + ++pos; has_digits = true; + } + } + if (has_digits && pos < s.size() && (s[pos] == 'e' || s[pos] == 'E')) { + ++pos; + if (pos < s.size() && (s[pos] == '+' || s[pos] == '-')) ++pos; + while (pos < s.size() && std::isdigit(static_cast(s[pos]))) ++pos; + } + if (has_digits && pos > start) { + std::string numstr = s.substr(start, pos - start); + try { + double val = std::stod(numstr); + return ConcoreValue::make_number(val); + } catch (...) { + throw std::runtime_error( + "Invalid concore payload: bad number '" + numstr + "'"); + } + } + pos = start; // backtrack + } + + throw std::runtime_error( + std::string("Invalid concore payload: unsupported literal at position ") + + std::to_string(pos)); +} + +/** + * Parses a complete Python literal string and returns a ConcoreValue. + * Trailing content after the value (other than whitespace) is an error. + */ +inline ConcoreValue parse_literal(const std::string& s) { + size_t pos = 0; + ConcoreValue v = parse_literal_value(s, pos); + skip_ws(s, pos); + if (pos != s.size()) + throw std::runtime_error( + "Invalid concore payload: unexpected trailing content"); + return v; +} + +/** + * Recursively extracts all numeric values from a ConcoreValue. + * Booleans convert to 1.0 / 0.0 (matching Python's int(True) / int(False)). + * Strings are skipped. + * Nested arrays are flattened. + */ +inline void flatten_numeric_impl(const ConcoreValue& v, std::vector& out) { + switch (v.type) { + case ConcoreValueType::NUMBER: + out.push_back(v.number); + break; + case ConcoreValueType::BOOL: + out.push_back(v.boolean ? 1.0 : 0.0); + break; + case ConcoreValueType::STRING: + // Skip non-numeric tokens + break; + case ConcoreValueType::ARRAY: + for (const auto& elem : v.array) + flatten_numeric_impl(elem, out); + break; + } +} + +inline std::vector flatten_numeric(const ConcoreValue& v) { + std::vector out; + flatten_numeric_impl(v, out); + return out; +} + +// --------------- parselist_double (full definition) ----------------- + inline std::vector parselist_double(const std::string& str) { - std::vector result; - std::vector tokens = parselist(str); - for (const auto& tok : tokens) { - result.push_back(std::stod(tok)); + std::string trimmed = stripstr(str); + if (trimmed.empty()) return {}; + try { + ConcoreValue v = parse_literal(trimmed); + return flatten_numeric(v); + } catch (...) { + // Fall back to the simple comma-split parser for edge cases + std::vector result; + if (trimmed.size() < 2) return result; + if (trimmed.front() == '[' || trimmed.front() == '(') { + std::vector tokens = parselist(trimmed); + for (const auto& tok : tokens) { + try { result.push_back(std::stod(tok)); } catch (...) {} + } + } + return result; } - return result; } /** From 3d35efd2d6f09f85af35a7d0fc99c9897395a1a8 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Tue, 24 Feb 2026 10:39:35 +0530 Subject: [PATCH 2/6] style: remove comments from new parser code --- concore.hpp | 13 ------------ concore_base.hpp | 54 +++++------------------------------------------- 2 files changed, 5 insertions(+), 62 deletions(-) diff --git a/concore.hpp b/concore.hpp index 5016a18..b8653cb 100644 --- a/concore.hpp +++ b/concore.hpp @@ -337,23 +337,10 @@ class Concore{ return concore_base::parselist_double(f); } - /** - * @brief Parses a Python-literal payload into a structured ConcoreValue. - * Supports numbers, booleans, strings, nested arrays, and tuples. - * Use this when you need the full parsed structure, not just doubles. - * @param f The input string to parse. - * @return A ConcoreValue representing the parsed literal. - * @throws std::runtime_error on malformed input. - */ concore_base::ConcoreValue parse_literal(string f){ return concore_base::parse_literal(f); } - /** - * @brief Recursively extracts all numeric values from a ConcoreValue. - * @param v The ConcoreValue to flatten. - * @return A flat vector of doubles. - */ vector flatten_numeric(const concore_base::ConcoreValue& v){ return concore_base::flatten_numeric(v); } diff --git a/concore_base.hpp b/concore_base.hpp index d03a0e0..3cb93e2 100644 --- a/concore_base.hpp +++ b/concore_base.hpp @@ -81,28 +81,10 @@ inline std::vector parselist(const std::string& str) { return result; } -/** - * Parses a double-valued list like "[0.0, 1.5, 2.3]" into a vector. - * Used by concore.hpp's read/write which work with numeric data. - * Now delegates to the full literal parser to handle mixed-type payloads - * (strings, booleans, nested lists, tuples) without crashing. - * See Issue #389. - */ -inline std::vector parselist_double(const std::string& str); // forward decl; defined after ConcoreValue - -// =================================================================== -// Python-Literal-Compatible Value Type and Parser (Issue #389) -// =================================================================== +inline std::vector parselist_double(const std::string& str); -/** - * Tag for ConcoreValue discriminated union. - */ enum class ConcoreValueType { NUMBER, BOOL, STRING, ARRAY }; -/** - * A recursive value type that mirrors Python's ast.literal_eval output. - * Supported: numbers, booleans, strings, and nested arrays / tuples. - */ struct ConcoreValue { ConcoreValueType type; double number; @@ -122,7 +104,7 @@ struct ConcoreValue { ConcoreValue cv; cv.type = ConcoreValueType::BOOL; cv.boolean = v; - cv.number = v ? 1.0 : 0.0; // Python: True == 1, False == 0 + cv.number = v ? 1.0 : 0.0; return cv; } static ConcoreValue make_string(const std::string& v) { @@ -139,8 +121,6 @@ struct ConcoreValue { } }; -// --------------- internal helpers (anonymous-namespace-like) -------- - inline void skip_ws(const std::string& s, size_t& pos) { while (pos < s.size() && std::isspace(static_cast(s[pos]))) ++pos; @@ -149,7 +129,7 @@ inline void skip_ws(const std::string& s, size_t& pos) { inline ConcoreValue parse_literal_value(const std::string& s, size_t& pos); inline ConcoreValue parse_literal_string(const std::string& s, size_t& pos) { - char quote = s[pos]; // ' or " + char quote = s[pos]; ++pos; std::string result; while (pos < s.size() && s[pos] != quote) { @@ -170,7 +150,7 @@ inline ConcoreValue parse_literal_string(const std::string& s, size_t& pos) { } if (pos >= s.size()) throw std::runtime_error("Invalid concore payload: unterminated string"); - ++pos; // skip closing quote + ++pos; return ConcoreValue::make_string(result); } @@ -190,10 +170,6 @@ inline ConcoreValue parse_literal_array(const std::string& s, size_t& pos) { throw std::runtime_error("Invalid concore payload: unterminated array/tuple"); } -/** - * Recursive descent parser entry for a single Python literal value. - * Advances `pos` past the consumed token. - */ inline ConcoreValue parse_literal_value(const std::string& s, size_t& pos) { skip_ws(s, pos); if (pos >= s.size()) @@ -201,34 +177,28 @@ inline ConcoreValue parse_literal_value(const std::string& s, size_t& pos) { char c = s[pos]; - // Array / Tuple if (c == '[' || c == '(') return parse_literal_array(s, pos); - // String if (c == '\'' || c == '"') return parse_literal_string(s, pos); - // Boolean True if (s.compare(pos, 4, "True") == 0 && (pos + 4 >= s.size() || !std::isalnum(static_cast(s[pos + 4])))) { pos += 4; return ConcoreValue::make_bool(true); } - // Boolean False if (s.compare(pos, 5, "False") == 0 && (pos + 5 >= s.size() || !std::isalnum(static_cast(s[pos + 5])))) { pos += 5; return ConcoreValue::make_bool(false); } - // None → treat as string "None" (no numeric equivalent) if (s.compare(pos, 4, "None") == 0 && (pos + 4 >= s.size() || !std::isalnum(static_cast(s[pos + 4])))) { pos += 4; return ConcoreValue::make_string("None"); } - // Number (int, float, negative, scientific notation) { size_t start = pos; if (pos < s.size() && (s[pos] == '+' || s[pos] == '-')) ++pos; @@ -257,7 +227,7 @@ inline ConcoreValue parse_literal_value(const std::string& s, size_t& pos) { "Invalid concore payload: bad number '" + numstr + "'"); } } - pos = start; // backtrack + pos = start; } throw std::runtime_error( @@ -265,10 +235,6 @@ inline ConcoreValue parse_literal_value(const std::string& s, size_t& pos) { std::to_string(pos)); } -/** - * Parses a complete Python literal string and returns a ConcoreValue. - * Trailing content after the value (other than whitespace) is an error. - */ inline ConcoreValue parse_literal(const std::string& s) { size_t pos = 0; ConcoreValue v = parse_literal_value(s, pos); @@ -279,12 +245,6 @@ inline ConcoreValue parse_literal(const std::string& s) { return v; } -/** - * Recursively extracts all numeric values from a ConcoreValue. - * Booleans convert to 1.0 / 0.0 (matching Python's int(True) / int(False)). - * Strings are skipped. - * Nested arrays are flattened. - */ inline void flatten_numeric_impl(const ConcoreValue& v, std::vector& out) { switch (v.type) { case ConcoreValueType::NUMBER: @@ -294,7 +254,6 @@ inline void flatten_numeric_impl(const ConcoreValue& v, std::vector& out out.push_back(v.boolean ? 1.0 : 0.0); break; case ConcoreValueType::STRING: - // Skip non-numeric tokens break; case ConcoreValueType::ARRAY: for (const auto& elem : v.array) @@ -309,8 +268,6 @@ inline std::vector flatten_numeric(const ConcoreValue& v) { return out; } -// --------------- parselist_double (full definition) ----------------- - inline std::vector parselist_double(const std::string& str) { std::string trimmed = stripstr(str); if (trimmed.empty()) return {}; @@ -318,7 +275,6 @@ inline std::vector parselist_double(const std::string& str) { ConcoreValue v = parse_literal(trimmed); return flatten_numeric(v); } catch (...) { - // Fall back to the simple comma-split parser for edge cases std::vector result; if (trimmed.size() < 2) return result; if (trimmed.front() == '[' || trimmed.front() == '(') { From 7849594d5c293783d05f768b345d30a8b2456070 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Mon, 2 Mar 2026 10:59:07 +0530 Subject: [PATCH 3/6] fix: restore original parselist_double doc comment --- concore_base.hpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/concore_base.hpp b/concore_base.hpp index 3cb93e2..f034f0e 100644 --- a/concore_base.hpp +++ b/concore_base.hpp @@ -81,6 +81,10 @@ inline std::vector parselist(const std::string& str) { return result; } +/** + * Parses a double-valued list like "[0.0, 1.5, 2.3]" into a vector. + * Used by concore.hpp's read/write which work with numeric data. + */ inline std::vector parselist_double(const std::string& str); enum class ConcoreValueType { NUMBER, BOOL, STRING, ARRAY }; From f9e17b90f7bf5cb2731e4d5a6aa7ff7b7256ca1a Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Mon, 2 Mar 2026 11:13:51 +0530 Subject: [PATCH 4/6] fix: apply Copilot review suggestions - README.md: clarify MATLAB/Verilog only support flat numeric arrays - concore_base.hpp: add #include and for portability - concore_base.hpp: fix keyword boundary checks to also exclude underscore - concore.hpp: add Doxygen docs for parse_literal and flatten_numeric --- README.md | 2 +- concore.hpp | 10 ++++++++++ concore_base.hpp | 11 ++++++++--- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 60c94e7..6c5c195 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ _concore_ enables composing studies from programs developed in different languag ## Wire Format -Concore payloads follow Python literal syntax compatible with `ast.literal_eval()`. All language implementations (Python, C++, Java, MATLAB) parse this shared format. Supported value types include: +Concore payloads follow Python literal syntax compatible with `ast.literal_eval()`. The Python, C++, and Java implementations parse this shared format; the MATLAB and Verilog implementations currently support only flat numeric arrays derived from it. Supported value types include: * **Numbers** — integers and floats, including scientific notation (e.g., `1e3`, `-2.5`) * **Booleans** — `True` / `False` (converted to `1.0` / `0.0` in numeric contexts) diff --git a/concore.hpp b/concore.hpp index b8653cb..5eb2b7e 100644 --- a/concore.hpp +++ b/concore.hpp @@ -337,10 +337,20 @@ class Concore{ return concore_base::parselist_double(f); } + /** + * @brief Parses a literal string into a ConcoreValue representation. + * @param f The input string to parse. + * @return A ConcoreValue obtained by parsing the input string. + */ concore_base::ConcoreValue parse_literal(string f){ return concore_base::parse_literal(f); } + /** + * @brief Flattens a ConcoreValue into a vector of numeric (double) values. + * @param v The ConcoreValue to flatten. + * @return A vector of double values obtained by flattening the input. + */ vector flatten_numeric(const concore_base::ConcoreValue& v){ return concore_base::flatten_numeric(v); } diff --git a/concore_base.hpp b/concore_base.hpp index f034f0e..b4d9689 100644 --- a/concore_base.hpp +++ b/concore_base.hpp @@ -14,6 +14,8 @@ #include #include #include +#include +#include namespace concore_base { @@ -188,17 +190,20 @@ inline ConcoreValue parse_literal_value(const std::string& s, size_t& pos) { return parse_literal_string(s, pos); if (s.compare(pos, 4, "True") == 0 && - (pos + 4 >= s.size() || !std::isalnum(static_cast(s[pos + 4])))) { + (pos + 4 >= s.size() || + (!std::isalnum(static_cast(s[pos + 4])) && s[pos + 4] != '_'))) { pos += 4; return ConcoreValue::make_bool(true); } if (s.compare(pos, 5, "False") == 0 && - (pos + 5 >= s.size() || !std::isalnum(static_cast(s[pos + 5])))) { + (pos + 5 >= s.size() || + (!std::isalnum(static_cast(s[pos + 5])) && s[pos + 5] != '_'))) { pos += 5; return ConcoreValue::make_bool(false); } if (s.compare(pos, 4, "None") == 0 && - (pos + 4 >= s.size() || !std::isalnum(static_cast(s[pos + 4])))) { + (pos + 4 >= s.size() || + (!std::isalnum(static_cast(s[pos + 4])) && s[pos + 4] != '_'))) { pos += 4; return ConcoreValue::make_string("None"); } From 2625e85afc2aeec36b7487201a3c75989df9e8a0 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Mon, 2 Mar 2026 11:18:32 +0530 Subject: [PATCH 5/6] style: format test_read_status.py with ruff --- tests/test_read_status.py | 46 ++++++++++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/tests/test_read_status.py b/tests/test_read_status.py index 54dc4b8..df3b7ef 100644 --- a/tests/test_read_status.py +++ b/tests/test_read_status.py @@ -13,6 +13,7 @@ # Helpers # --------------------------------------------------------------------------- + class DummyZMQPort: """Minimal stand-in for ZeroMQPort used in ZMQ read tests.""" @@ -33,14 +34,16 @@ def recv_json_with_retry(self): # File-based read tests # --------------------------------------------------------------------------- + class TestReadFileSuccess: """read() on a valid file returns (data, True) with SUCCESS status.""" @pytest.fixture(autouse=True) def setup(self, temp_dir, monkeypatch): import concore + self.concore = concore - monkeypatch.setattr(concore, 'delay', 0) + monkeypatch.setattr(concore, "delay", 0) # Create ./in1/ym with valid data: [simtime, value] in_dir = os.path.join(temp_dir, "in1") @@ -48,7 +51,7 @@ def setup(self, temp_dir, monkeypatch): with open(os.path.join(in_dir, "ym"), "w") as f: f.write("[10, 3.14]") - monkeypatch.setattr(concore, 'inpath', os.path.join(temp_dir, "in")) + monkeypatch.setattr(concore, "inpath", os.path.join(temp_dir, "in")) def test_returns_data_and_true(self): data, ok = self.concore.read(1, "ym", "[0, 0.0]") @@ -66,10 +69,11 @@ class TestReadFileMissing: @pytest.fixture(autouse=True) def setup(self, temp_dir, monkeypatch): import concore + self.concore = concore - monkeypatch.setattr(concore, 'delay', 0) + monkeypatch.setattr(concore, "delay", 0) # Point to a directory that does NOT have the file - monkeypatch.setattr(concore, 'inpath', os.path.join(temp_dir, "in")) + monkeypatch.setattr(concore, "inpath", os.path.join(temp_dir, "in")) def test_returns_default_and_false(self): data, ok = self.concore.read(1, "nonexistent", "[0, 0.0]") @@ -86,15 +90,16 @@ class TestReadFileParseError: @pytest.fixture(autouse=True) def setup(self, temp_dir, monkeypatch): import concore + self.concore = concore - monkeypatch.setattr(concore, 'delay', 0) + monkeypatch.setattr(concore, "delay", 0) in_dir = os.path.join(temp_dir, "in1") os.makedirs(in_dir, exist_ok=True) with open(os.path.join(in_dir, "ym"), "w") as f: f.write("NOT_VALID_PYTHON{{{") - monkeypatch.setattr(concore, 'inpath', os.path.join(temp_dir, "in")) + monkeypatch.setattr(concore, "inpath", os.path.join(temp_dir, "in")) def test_returns_default_and_false(self): data, ok = self.concore.read(1, "ym", "[0, 0.0]") @@ -111,8 +116,9 @@ class TestReadFileRetriesExceeded: @pytest.fixture(autouse=True) def setup(self, temp_dir, monkeypatch): import concore + self.concore = concore - monkeypatch.setattr(concore, 'delay', 0) + monkeypatch.setattr(concore, "delay", 0) # Create an empty file in_dir = os.path.join(temp_dir, "in1") @@ -120,7 +126,7 @@ def setup(self, temp_dir, monkeypatch): with open(os.path.join(in_dir, "ym"), "w") as f: pass # empty - monkeypatch.setattr(concore, 'inpath', os.path.join(temp_dir, "in")) + monkeypatch.setattr(concore, "inpath", os.path.join(temp_dir, "in")) def test_returns_default_and_false(self): data, ok = self.concore.read(1, "ym", "[0, 0.0]") @@ -135,12 +141,14 @@ def test_last_read_status_is_retries_exceeded(self): # ZMQ read tests # --------------------------------------------------------------------------- + class TestReadZMQSuccess: """Successful ZMQ read returns (data, True).""" @pytest.fixture(autouse=True) def setup(self, monkeypatch): import concore + self.concore = concore self.original_ports = concore.zmq_ports.copy() yield @@ -164,6 +172,7 @@ class TestReadZMQTimeout: @pytest.fixture(autouse=True) def setup(self, monkeypatch): import concore + self.concore = concore self.original_ports = concore.zmq_ports.copy() yield @@ -185,6 +194,7 @@ class TestReadZMQError: @pytest.fixture(autouse=True) def setup(self, monkeypatch): import concore + self.concore = concore self.original_ports = concore.zmq_ports.copy() yield @@ -193,6 +203,7 @@ def setup(self, monkeypatch): def test_zmq_error_returns_default_and_false(self): import zmq + dummy = DummyZMQPort(raise_on_recv=zmq.error.ZMQError("test error")) self.concore.zmq_ports["test_port"] = dummy @@ -205,21 +216,23 @@ def test_zmq_error_returns_default_and_false(self): # Backward compatibility # --------------------------------------------------------------------------- + class TestReadBackwardCompatibility: """Legacy callers can use isinstance check on the result.""" @pytest.fixture(autouse=True) def setup(self, temp_dir, monkeypatch): import concore + self.concore = concore - monkeypatch.setattr(concore, 'delay', 0) + monkeypatch.setattr(concore, "delay", 0) in_dir = os.path.join(temp_dir, "in1") os.makedirs(in_dir, exist_ok=True) with open(os.path.join(in_dir, "ym"), "w") as f: f.write("[10, 42.0]") - monkeypatch.setattr(concore, 'inpath', os.path.join(temp_dir, "in")) + monkeypatch.setattr(concore, "inpath", os.path.join(temp_dir, "in")) def test_legacy_unpack_pattern(self): """The recommended migration pattern works correctly.""" @@ -245,17 +258,24 @@ def test_tuple_unpack(self): # last_read_status exposed on module # --------------------------------------------------------------------------- + class TestLastReadStatusExposed: """concore.last_read_status is publicly accessible.""" def test_attribute_exists(self): import concore - assert hasattr(concore, 'last_read_status') + + assert hasattr(concore, "last_read_status") def test_initial_value_is_success(self): import concore + # Before any read, default is SUCCESS assert concore.last_read_status in ( - "SUCCESS", "FILE_NOT_FOUND", "TIMEOUT", - "PARSE_ERROR", "EMPTY_DATA", "RETRIES_EXCEEDED", + "SUCCESS", + "FILE_NOT_FOUND", + "TIMEOUT", + "PARSE_ERROR", + "EMPTY_DATA", + "RETRIES_EXCEEDED", ) From 37082417a133d688bd34a7287f8059f49ca2ec43 Mon Sep 17 00:00:00 2001 From: Ganesh Patil <7030871503ganeshpatil@gmail.com> Date: Mon, 2 Mar 2026 11:21:40 +0530 Subject: [PATCH 6/6] style: fix ruff lint errors in test_read_status.py --- tests/test_read_status.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_read_status.py b/tests/test_read_status.py index df3b7ef..29e6434 100644 --- a/tests/test_read_status.py +++ b/tests/test_read_status.py @@ -6,7 +6,6 @@ import os import pytest -import numpy as np # --------------------------------------------------------------------------- @@ -123,7 +122,7 @@ def setup(self, temp_dir, monkeypatch): # Create an empty file in_dir = os.path.join(temp_dir, "in1") os.makedirs(in_dir, exist_ok=True) - with open(os.path.join(in_dir, "ym"), "w") as f: + with open(os.path.join(in_dir, "ym"), "w") as _f: pass # empty monkeypatch.setattr(concore, "inpath", os.path.join(temp_dir, "in"))