From d7a60a9187c0fc5fd6f007846aadc1c6a6ec4e54 Mon Sep 17 00:00:00 2001 From: Omer Shamash Date: Mon, 9 Mar 2020 16:34:00 +0000 Subject: [PATCH 1/7] MRI v1 Initial commit (WIP). Proper release when ready --- .gitignore | 10 +++ doc/arch.md | 0 include/mri.h | 111 +++++++++++++++++++++++++++++++++ src/mri_backend.cpp | 145 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 266 insertions(+) create mode 100644 .gitignore create mode 100644 doc/arch.md create mode 100644 include/mri.h create mode 100644 src/mri_backend.cpp diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8381a0d --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +.cproject +.project +*.idea/ +.pydevproject +*.pyc +.vscode/ +GPATH +GRTAGS +GTAGS + diff --git a/doc/arch.md b/doc/arch.md new file mode 100644 index 0000000..e69de29 diff --git a/include/mri.h b/include/mri.h new file mode 100644 index 0000000..da9dadf --- /dev/null +++ b/include/mri.h @@ -0,0 +1,111 @@ +#ifndef _MRI_H_ +#define _MRI_H_ + +#include +#include + +/* defines */ +#define MRI_MAX_SLOT_STR_SIZE (256) +#define MRI_USERDATA_SIZE (32) + +#define MRI_SLOT_FLAG_CONST (1 << 0) /* TODO(omer): Unused for now */ +#define MRI_SLOT_FLAG_RATE (1 << 1) /* TODO(omer): Unused for now */ +#define MRI_SLOT_FLAG_PK (1 << 2) +#define MRI_SLOT_FLAG_HIDDEN (1 << 3) + +/* typedefs */ +typedef uint8_t mri_byte_t; +typedef const char *cstring_t; + +typedef struct mri_iter_state { + size_t iteration_count; /* Iteration count (incremented by MRI) */ + mri_byte_t userdata[MRI_USERDATA_SIZE]; /* User provided data to help iteration */ +} mri_iter_state_t; + +/* Reflects result of capturing a slot */ +typedef struct mri_capture_sample { + uint32_t ns_timestamp; /* Data timestamp (monotonic nanoseconds) */ + mri_byte_t sample[MRI_MAX_SLOT_STR_SIZE]; /* Data captured via calls to the formatter */ +} mri_capture_sample_t; + +/* Provided formatters (multi-formatters) */ +typedef struct {} mri_hex_t; /* Hexadecimal representation */ +typedef struct {} mri_number_t; /* Plain normal numeric data */ +typedef struct {} mri_buffer_t; /* Memory address dump in bytes */ +typedef struct {} mri_string_t; /* ASCII encoded human string */ +typedef struct {} mri_timespec_t; /* timespec (tv_sec, tv_usec) */ +typedef struct {} mri_epoch_time_t; /* uint64_t seconds since epoch */ +typedef struct {} mri_elapsed_time_t; /* uint64_t monotonic diff */ + +/* Provided shapers (multi-shapers) */ +typedef struct {} mri_rate_shaper_t; /* Numeric slot rate per second */ + +/* Callbacks (TBD) */ + +/* Input: slot-data, size-of-slot, output-string */ +typedef int (*mri_formatter_cb)(void *, size_t, char *); +/* Input: type-data, iteration-state-data, output-memory */ +typedef int (*mri_iterator_cb)(void *, mri_iter_state_t *, void *); +/* Input: registered-context, cli-input, size-of-input */ +typedef int (*mri_config_change_cb)(void *, cstring_t, size_t); +/* Input: list-of-samples, amount-of-samples, output-string */ +typedef int (*mri_shaper_cb)(mri_capture_sample_t *, size_t, char *); + +/* prototypes */ + +/* Notes: + * Each instance can only be linked to one domain (! consider !) + * Instance is global to process having thread-oriented CB support + */ +int mri_init(const char *domain); + +/* Formatting apart from provided-formatters section */ +int _mri_create_formatter(cstring_t name, mri_formatter_cb callback); + +/* TODO(omer): Add shaper configuration (amount of samples, scope and more */ + +/* Shaping apart from provided-shapers section */ +int _mri_create_shaper(cstring_t name, mri_shaper_cb callback); + +/* Registering types and data to MRI */ +int _mri_create_type(cstring_t name, size_t size); +int _mri_type_add_slot(cstring_t type, cstring_t name, cstring_t formatter, size_t offset, size_t size, int slot_flags); +int _mri_type_add_vslot(cstring_t type, cstring_t name, mri_formatter_cb callback); +int _mri_type_add_shaper(cstring_t type, cstring_t name, cstring_t formatter, size_t offset, size_t size, cstring_t shaper); + +/* Current suggestion: + * when someone registers data on path, we capture his thread-id and + * allow him to poll on an event object that is related to his thread-id + * + * mri_get_eventloop_fd() will allow user to allocate time for MRI in any bitch-thread + * mri_get_notification_fd() will allow to get the thread-bound event-fd to poll in relevant thread + */ +int mri_get_eventloop_fd(); +int mri_get_notification_fd(); + +/* Registering data of defined types to MRI */ +int _mri_register(cstring_t path, cstring_t type, void *object, mri_iterator_cb callback); +int mri_register_config(cstring_t path, void *context, mri_config_change_cb callback); +int mri_unregister(cstring_t path); + +/* Formalize API using macros */ +#define mri_create_type(c_type) \ + _mri_create_type(#c_type, sizeof(c_type)) +#define mri_type_add_slot(c_type, slot, format, flags) \ + _mri_type_add_slot(#c_type, #slot, #format, (size_t) &(((c_type *) 0)->slot), sizeof(((c_type *) 0)->slot), flags) +#define mri_type_add_vslot(c_type, slot, vslot_format) \ + _mri_type_add_vslot(#c_type, slot, vslot_foramt) +#define mri_register(path, c_type, object, callback) \ + _mri_register(path, #c_type, object, callback) +#define mri_create_formatter(c_type, callback) \ + _mri_create_fromatter(#c_type, callback) +#define mri_create_shaper(c_type, callback) \ + _mri_create_shaper(#c_type, callback) + +#define mri_type_add_shaper_slot(c_type, slot, format, shaper) \ + _mri_type_add_shaper(#c_type, #slot, #format, (size_t) &(((c_type *) 0)->slot), sizeof(((c_type *) 0)->slot), #shaper) + +#define mri_type_add_rate(c_type, slot) \ + mri_type_add_shaper(c_type, slot, mri_number_t, mri_rate_shaper_t) + +#endif /* _MRI_H_ */ diff --git a/src/mri_backend.cpp b/src/mri_backend.cpp new file mode 100644 index 0000000..a725228 --- /dev/null +++ b/src/mri_backend.cpp @@ -0,0 +1,145 @@ +#include +#include +#include +#include + +#include "../include/mri.h" +#include "../deps/ordered_map.h" + +/* TODO(omer): Change to something real */ +#define mri_assert(cond) (void) 0 + +/* Declarations (should move soon) */ +struct vslot; +struct xslot; +struct xtype; +struct xflat_type; +struct xpath_node; + +/* TODO(omer): Remove */ +namespace std { struct string_view; }; + +template +using mri_slot_map_t = tsl::ordered_map>; +using mri_capture_map_t = std::unordered_map>; +using mri_subpath_map_t = tsl::ordered_map>; +using mri_formatter_map_t = std::unordered_map; +using mri_shaper_map_t = std::unordered_map; + +/** + * MRI_SLOT + * Member of a registered user-defined data structure + */ +struct mri_slot { + std::string m_name; + size_t m_offset; + xflat_type const &m_type; + bool m_is_hidden; + + template + mri_slot(string_t &&name, size_t offset, xflat_type const &slt_type, bool hidden) + : m_name(std::forward(name)) + , m_offset(offset) + , m_type(slt_type) + , m_is_hidden(hidden) + {} +}; + +/** + * XFLAT_TYPE + * Type which can be formatted into string + * All slots are flat-typed to avoid complexity + */ +struct xflat_type { + std::string m_name; /*!< Name of this flatten-type (unique) */ + size_t m_size; /*!< Size of the type (used by m_format_cb) */ + mri_formatter_cb m_format_cb; /*!< Format callback (transform to string) */ + + template + xflat_type(string_t &&name, size_t size, mri_formatter_cb callback) + : m_name(std::forward(name)) + , m_size(size), + , m_format_cb(callback) + {} +}; + +/** + * XTYPE + */ +struct xtype { + std::string m_name; /*!< Name of this X-Type (unique) */ + mri_slot_map_t m_slots; /*!< Ordered mapping of all type's slots */ + struct mri_slot *m_private_key; /*!< Explicit pointer to the type's PK slot */ + mri_capture_map_t m_slot_capture_map; /*!< Externalize the per-slot data captures */ + + template + xtype(string_t &&name) + : m_name(std::forward(name)) + {} + + /** + * Create a slot (data-member) + * @param name - name of the new slot + * @param offset - byte offset in data struct (offsetof) + * @param type - name of the member type (used to fetch formatter) + * @param flags - bitwise flags of slot options (pk, rate, hidden, etc...) + */ + void add_slot(std::string_view name, size_t offset, std::string_view type, int flags=0); + + /** + * Create a virtual slot -> + * 1. Creates new xflat_type using the custom-callback + * 2. Adds a new slot with type of the newly created xflat_type + * + * @param name - name of the new virtual slot + * @param callback - user-defined function to format vslot + */ + void add_vslot(std::string_view name, mri_formatter_cb callback); +}; + +/** + * xpath_node + * Chains of nodes defining path (/ is root) + */ +struct xpath_node { + /* General definition of a node */ + std::string m_name; /*>! Name of node (name unique per parent) */ + mri_subpath_map_t m_sub_nodes; /*>! Mapping of sub-nodes (each is unique) */ + xtype *m_type; /*>! Type coresponding with the node's data */ + xpath_node *m_parent; /*>! Reverse reference to the parent's node */ + + /* User-data (specified on registration) */ + tid_t m_tid; /*>! Thread whom registered node (origin) */ + void *m_xdata; /*>! User-defined data (node data internal) */ + mri_iterator_cb m_xdata_iterator; /*>! User-defined data iterator (tableview) */ + + /** + * Notes: + * 1. No type = empty node, path-building + * 2. No parent = root node, others should not do this + * 3. No data = only allowed for empty node + * 4. Iterator = optional, data can be flat + * 5. Name cannot contain / (except root) to allow path traversal + * 6. Path is defined as ///.../ + */ + template + xpath_node( string_t &&name, + xtype *type, + xpath_node *parent, + void *xdata, + mri_iterator_cb data_iter) + : m_name(std::forward(name)) + , m_sub_nodes() + , m_type(type) + , m_parent(parent) + , m_tid(_gettid()) + , m_xdata(xdata) + , m_xdata_iterator(data_iter) + { mri_assert(parent || ! data); } +}; + +/* Global variables */ +static mri_shaper_map_t mri_shapers; +static mri_formatter_map_t mri_formatters; +static xpath_node mri_root ("/", nullptr, nullptr, nullptr, nullptr); + From fa8c579ff2530845126627a9b3578d14f1fd0dd0 Mon Sep 17 00:00:00 2001 From: Omer Shamash Date: Tue, 10 Mar 2020 14:41:11 +0000 Subject: [PATCH 2/7] MRI: Kfir reviews like a mofo --- include/mri.h | 15 +++++++++++-- include/mri.hpp | 52 +++++++++++++++++++++++++++++++++++++++++++++ src/mri_backend.cpp | 11 +++++++++- 3 files changed, 75 insertions(+), 3 deletions(-) create mode 100644 include/mri.hpp diff --git a/include/mri.h b/include/mri.h index da9dadf..bcfae73 100644 --- a/include/mri.h +++ b/include/mri.h @@ -12,6 +12,11 @@ #define MRI_SLOT_FLAG_RATE (1 << 1) /* TODO(omer): Unused for now */ #define MRI_SLOT_FLAG_PK (1 << 2) #define MRI_SLOT_FLAG_HIDDEN (1 << 3) + +typedef enum mri_iter_action { + MRI_ITER_STOP, + MRI_ITER_CONTINUE +} mri_iter_action_t; /* typedefs */ typedef uint8_t mri_byte_t; @@ -45,7 +50,7 @@ typedef struct {} mri_rate_shaper_t; /* Numeric slot rate per second */ /* Input: slot-data, size-of-slot, output-string */ typedef int (*mri_formatter_cb)(void *, size_t, char *); /* Input: type-data, iteration-state-data, output-memory */ -typedef int (*mri_iterator_cb)(void *, mri_iter_state_t *, void *); +typedef mri_iter_action_t (*mri_iterator_cb)(void *, mri_iter_state_t *, void **); /* Input: registered-context, cli-input, size-of-input */ typedef int (*mri_config_change_cb)(void *, cstring_t, size_t); /* Input: list-of-samples, amount-of-samples, output-string */ @@ -53,9 +58,15 @@ typedef int (*mri_shaper_cb)(mri_capture_sample_t *, size_t, char *); /* prototypes */ -/* Notes: +/** + * mri_init() + * + * Notes: * Each instance can only be linked to one domain (! consider !) * Instance is global to process having thread-oriented CB support + * + * @param domain - root-path, shared, kind of like catagory (routing, system-events, em, etc...) + * @returns C-Style boolean (0 = OK, <0 = ERROR) (-ERRNO) */ int mri_init(const char *domain); diff --git a/include/mri.hpp b/include/mri.hpp new file mode 100644 index 0000000..9adc460 --- /dev/null +++ b/include/mri.hpp @@ -0,0 +1,52 @@ +#include + +extern "C" { + #include "mri.h" +}; + +/** + * Generic template to form any iteration of given container + * + * @tparam container_t Container-type to register. + * @tparam iterator_t Value Iterator-type to register. + * @tparam value_t Value-type to register. + * + * Other parameters as defined in typedef mri_iterator_cb + */ +template < typename container_t, + typename iterator_t = typename container_t::iterator, + typename value_t = typename container_t::value_type > +int generic_mri_iterator_cb(void *container, mri_iter_state_t *state, void **mem) { + /* We are getting the templated type as data */ + auto const &data_view = *((container_t *) container); + + /* Check for first run */ + if (0 == state->iteration_count) { + /* Deep-copy since iterators don't have copy .ctor */ + auto begin_iterator = data_view.begin(); + + /** + * Notice we disable the warning because normally one cannot copy-construct an iterator + * This is purely a hack we use to avoid the need for having an in-container detecated iterator + */ + #pragma GCC diagnostic push + #if __GNUC__ >= 8 + #pragma GCC diagnostic ignored "-Wclass-memaccess" + #endif + std::memcpy(&state->userdata, &begin_iterator, sizeof(iterator_t)); + #pragma GCC diagnostic pop + } + + /* Parse private data as iterator (see above) */ + auto &iterator = *((iterator_t *) &state->userdata); + + // If we are at the last argument -> restart iterator + if (data_view.end() == iterator) { + return MRI_ITER_STOP; + } + + /* Set row memory from iterator using the operator */ + *mem = &(* iterator++); /* pointer to data reference */ + + return MRI_ITER_CONTINUE; +} diff --git a/src/mri_backend.cpp b/src/mri_backend.cpp index a725228..40f6bf5 100644 --- a/src/mri_backend.cpp +++ b/src/mri_backend.cpp @@ -3,7 +3,12 @@ #include #include -#include "../include/mri.h" +/* TODO(omer): Fix include path */ +extern "C" { + #include "../include/mri.h" +// #include "dn_common/general.h" +}; + #include "../deps/ordered_map.h" /* TODO(omer): Change to something real */ @@ -63,8 +68,12 @@ struct xflat_type { {} }; +/* TODO(omer): Consider enabling a flat-parse for an xtype (slot, slot, slot, ...) */ + /** * XTYPE + * Type which is composed from multiple slots (complex type) + * Those types are usually being registered to one or more xpath_node */ struct xtype { std::string m_name; /*!< Name of this X-Type (unique) */ From 178f481aae9b524dedb4ba3d348a364e7246aa23 Mon Sep 17 00:00:00 2001 From: Omer Shamash Date: Mon, 16 Mar 2020 17:38:52 +0000 Subject: [PATCH 3/7] MRI: v1: Data moduling done right --- include/mri.h | 15 ++- include/mri.hpp | 2 +- src/mri_backend.cpp | 318 ++++++++++++++++++++++++++++++++++++-------- 3 files changed, 269 insertions(+), 66 deletions(-) diff --git a/include/mri.h b/include/mri.h index bcfae73..39e0be5 100644 --- a/include/mri.h +++ b/include/mri.h @@ -4,6 +4,9 @@ #include #include +/* Logger related */ +#include "mri_logging.h" + /* defines */ #define MRI_MAX_SLOT_STR_SIZE (256) #define MRI_USERDATA_SIZE (32) @@ -44,7 +47,7 @@ typedef struct {} mri_elapsed_time_t; /* uint64_t monotonic diff */ /* Provided shapers (multi-shapers) */ typedef struct {} mri_rate_shaper_t; /* Numeric slot rate per second */ - + /* Callbacks (TBD) */ /* Input: slot-data, size-of-slot, output-string */ @@ -68,7 +71,7 @@ typedef int (*mri_shaper_cb)(mri_capture_sample_t *, size_t, char *); * @param domain - root-path, shared, kind of like catagory (routing, system-events, em, etc...) * @returns C-Style boolean (0 = OK, <0 = ERROR) (-ERRNO) */ -int mri_init(const char *domain); +int mri_init(cstring_t domain); /* Formatting apart from provided-formatters section */ int _mri_create_formatter(cstring_t name, mri_formatter_cb callback); @@ -79,10 +82,10 @@ int _mri_create_formatter(cstring_t name, mri_formatter_cb callback); int _mri_create_shaper(cstring_t name, mri_shaper_cb callback); /* Registering types and data to MRI */ -int _mri_create_type(cstring_t name, size_t size); -int _mri_type_add_slot(cstring_t type, cstring_t name, cstring_t formatter, size_t offset, size_t size, int slot_flags); -int _mri_type_add_vslot(cstring_t type, cstring_t name, mri_formatter_cb callback); -int _mri_type_add_shaper(cstring_t type, cstring_t name, cstring_t formatter, size_t offset, size_t size, cstring_t shaper); +int _mri_create_type(cstring_t, size_t); +int _mri_type_add_slot(cstring_t, cstring_t, cstring_t, size_t, size_t, int); +int _mri_type_add_vslot(cstring_t, cstring_t, mri_formatter_cb); +int _mri_type_add_shaper(cstring_t, cstring_t, cstring_t, size_t, size_t); /* Current suggestion: * when someone registers data on path, we capture his thread-id and diff --git a/include/mri.hpp b/include/mri.hpp index 9adc460..c354888 100644 --- a/include/mri.hpp +++ b/include/mri.hpp @@ -27,7 +27,7 @@ int generic_mri_iterator_cb(void *container, mri_iter_state_t *state, void **mem /** * Notice we disable the warning because normally one cannot copy-construct an iterator - * This is purely a hack we use to avoid the need for having an in-container detecated iterator + * This is purely a hack we use to avoid the need for having an in-container dedicated iterator */ #pragma GCC diagnostic push #if __GNUC__ >= 8 diff --git a/src/mri_backend.cpp b/src/mri_backend.cpp index 40f6bf5..deeb283 100644 --- a/src/mri_backend.cpp +++ b/src/mri_backend.cpp @@ -1,70 +1,88 @@ +#include #include #include -#include #include -/* TODO(omer): Fix include path */ +/* TODO(omer): Fix include paths */ extern "C" { #include "../include/mri.h" // #include "dn_common/general.h" -}; +} -#include "../deps/ordered_map.h" +#include "general.hpp" +//#include "../deps/ordered_map.h" /* TODO(omer): Change to something real */ #define mri_assert(cond) (void) 0 +/* gettid() is undefined in glibc, use this instead */ +#include +#include +typedef pid_t tid_t; +static tid_t _gettid() { return syscall(SYS_gettid); } + /* Declarations (should move soon) */ -struct vslot; -struct xslot; struct xtype; -struct xflat_type; +struct xformat; +struct xshaper; +struct mri_slot; struct xpath_node; -/* TODO(omer): Remove */ -namespace std { struct string_view; }; - -template -using mri_slot_map_t = tsl::ordered_map>; -using mri_capture_map_t = std::unordered_map>; -using mri_subpath_map_t = tsl::ordered_map>; -using mri_formatter_map_t = std::unordered_map; -using mri_shaper_map_t = std::unordered_map; +using mri_slot_map_t = std::map; +using mri_subpath_map_t = std::map; +using mri_capture_map_t = std::unordered_map>; +using mri_formatter_map_t = std::unordered_map; +using mri_shaper_map_t = std::unordered_map; +using mri_xtype_map_t = std::unordered_map; -/** - * MRI_SLOT - * Member of a registered user-defined data structure - */ -struct mri_slot { +struct xformat { std::string m_name; - size_t m_offset; - xflat_type const &m_type; - bool m_is_hidden; + mri_formatter_cb m_formatter; template - mri_slot(string_t &&name, size_t offset, xflat_type const &slt_type, bool hidden) - : m_name(std::forward(name)) - , m_offset(offset) - , m_type(slt_type) - , m_is_hidden(hidden) + xformat(string_t &&name, mri_formatter_cb formatter) + : m_name(std::forward(name)) + , m_formatter(formatter) + {} +}; + +struct xshaper { + std::string m_name; + mri_shaper_cb m_shaper; + + template + xshaper(string_t &&name, mri_shaper_cb shaper) + : m_name(std::forward(name)) + , m_shaper(shaper) {} }; /** - * XFLAT_TYPE - * Type which can be formatted into string - * All slots are flat-typed to avoid complexity + * MRI_SLOT + * Member of a registered user-defined data structure */ -struct xflat_type { - std::string m_name; /*!< Name of this flatten-type (unique) */ - size_t m_size; /*!< Size of the type (used by m_format_cb) */ - mri_formatter_cb m_format_cb; /*!< Format callback (transform to string) */ +struct mri_slot { + std::string m_name; /*!< Name of the slot (unique per-xtype) */ + size_t m_offset; /*!< Offset of the slot's data in xtype */ + size_t m_size; /*!< Size of the slot (used by formatter) */ + mri_formatter_cb m_formatter; /*!< Reference to formatter (int, str, ...) */ + mri_shaper_cb m_shaper; /*!< Pointer to slot data-samples shaper_cb */ + bool m_is_hidden; /*!< Indicator for external usage (to-show) */ + /* TODO(omer): Find a better way (m_shaper doubles as is-slot-shaper indicator) */ + /* Note: m_shaper will be filled externally for xpath_node::xtype::dump to use */ template - xflat_type(string_t &&name, size_t size, mri_formatter_cb callback) - : m_name(std::forward(name)) - , m_size(size), - , m_format_cb(callback) + mri_slot( string_t &&name, + size_t offset, + size_t size, + mri_formatter_cb formatter, + bool hidden) + : m_name(std::forward(name)) + , m_offset(offset) + , m_size(size) + , m_formatter(formatter) + , m_shaper(nullptr) + , m_is_hidden(hidden) {} }; @@ -76,34 +94,31 @@ struct xflat_type { * Those types are usually being registered to one or more xpath_node */ struct xtype { - std::string m_name; /*!< Name of this X-Type (unique) */ - mri_slot_map_t m_slots; /*!< Ordered mapping of all type's slots */ - struct mri_slot *m_private_key; /*!< Explicit pointer to the type's PK slot */ - mri_capture_map_t m_slot_capture_map; /*!< Externalize the per-slot data captures */ + std::string m_name; /*!< Name of this X-Type (unique) */ + size_t m_size; /*!< Size needed to allocate the type */ + mri_slot_map_t m_slots; /*!< Ordered mapping of all type's slots */ + struct mri_slot *m_private_key; /*!< Explicit pointer to the type's PK slot */ + mri_capture_map_t m_slot_capture_map; /*!< Externalize the per-slot data captures */ template - xtype(string_t &&name) - : m_name(std::forward(name)) + xtype(string_t &&name, size_t size) + : m_name(std::forward(name)) + , m_size(size) + , m_slots() + , m_private_key(nullptr) + , m_slot_capture_map() {} /** * Create a slot (data-member) * @param name - name of the new slot * @param offset - byte offset in data struct (offsetof) - * @param type - name of the member type (used to fetch formatter) + * @param size - size in bytes of contained data (sizeof) + * @param formatter - format callback for the new slot (to_string) * @param flags - bitwise flags of slot options (pk, rate, hidden, etc...) + * @return C-Style boolean (0 = OK, <0 = ERROR) (-ERRNO) */ - void add_slot(std::string_view name, size_t offset, std::string_view type, int flags=0); - - /** - * Create a virtual slot -> - * 1. Creates new xflat_type using the custom-callback - * 2. Adds a new slot with type of the newly created xflat_type - * - * @param name - name of the new virtual slot - * @param callback - user-defined function to format vslot - */ - void add_vslot(std::string_view name, mri_formatter_cb callback); + int add_slot(cstring_t name, size_t offset, size_t size, mri_formatter_cb formatter, int flags); }; /** @@ -137,7 +152,7 @@ struct xpath_node { xpath_node *parent, void *xdata, mri_iterator_cb data_iter) - : m_name(std::forward(name)) + : m_name(std::forward(name)) , m_sub_nodes() , m_type(type) , m_parent(parent) @@ -148,7 +163,192 @@ struct xpath_node { }; /* Global variables */ +static mri_xtype_map_t mri_xtypes; static mri_shaper_map_t mri_shapers; static mri_formatter_map_t mri_formatters; static xpath_node mri_root ("/", nullptr, nullptr, nullptr, nullptr); +#define MRI_MAP_INSERT(map, key, ...) \ + /* Note: ignore potential copy, even by using std::move, compiler optimization will fix this */ \ + /* Note: usage of m_name.c_str() as the map-key allows us to avoid another string allocation */ \ + decltype(map)::mapped_type __temp_cls (key, ##__VA_ARGS__); \ + auto&& [iter, did_insert] = map.try_emplace(__temp_cls.m_name.c_str(), std::move(__temp_cls)) \ + +#define MRI_SAFE_MAP_INSERTION(exists_err, map_data_t, map, key, ...) \ + MRI_MAP_INSERT(map, key, ##__VA_ARGS__); \ + \ + if (likely(did_insert)) return 0; /* Bailout if all worked */ \ + \ + if (map.end() == iter) { \ + MRI_LOG(MRI_ERROR, "Failed to insert " #key " %s " #map_data_t " (OOM)", key); \ + return -ENOMEM; \ + } \ + \ + MRI_LOG(MRI_WARNING, #map_data_t " %s already exists", key); \ + return exists_err + +int xtype::add_slot(cstring_t name, size_t offset, size_t size, mri_formatter_cb formatter, int flags) { + (void) name; + (void) offset; + (void) size; + (void) formatter; + (void) flags; + + return 0; +} + +extern "C" { + + int mri_init(cstring_t domain) { + MRI_SAFE_MAP_INSERTION( 0, + xpath_node, + mri_root.m_sub_nodes, + domain, + nullptr, + &mri_root, + nullptr, + nullptr); + } + + /* --------------------------------------------------------------------------------------------- + * Delegators to map user-defined format/shaper + * --------------------------------------------------------------------------------------------- + */ + + int _mri_create_formatter(cstring_t formatter, mri_formatter_cb callback) { + MRI_SAFE_MAP_INSERTION( -EEXIST, + xformat, + mri_formatters, + formatter, + callback); + } + + int _mri_create_shaper(cstring_t shaper, mri_shaper_cb callback) { + MRI_SAFE_MAP_INSERTION( -EEXIST, + xshaper, + mri_shapers, + shaper, + callback); + } + + /* --------------------------------------------------------------------------------------------- + * Custom type creation functions + * --------------------------------------------------------------------------------------------- + */ + + int _mri_create_type(cstring_t type, size_t size) { + MRI_SAFE_MAP_INSERTION( 0, + xtype, + mri_xtypes, + type, + size); + } + + + int _mri_type_add_slot( cstring_t type, + cstring_t name, + cstring_t formatter, + size_t offset, + size_t size, + int slot_flags) { + auto xtype_iter = mri_xtypes.find(type); + if (unlikely(mri_xtypes.end() == xtype_iter)) { + MRI_LOG(MRI_ERROR, "xtype %s does not exist, better luck next time", type); + return -EINVAL; + } + + auto xformat_iter = mri_formatters.find(formatter); + if (unlikely(mri_formatters.end() == xformat_iter)) { + MRI_LOG(MRI_ERROR, "xformat %s does not exist, better luck next time", formatter); + return -EINVAL; + } + + auto &mri_type = xtype_iter->second; + auto const &format = xformat_iter->second; + return mri_type.add_slot(name, offset, size, format.m_formatter, slot_flags); + } + + /** + * Create a virtual slot -> + * 1. Creates new xformat using the custom-callback + * 2. Adds a new slot with type of the newly created xformat + * + * @param name - name of the new virtual slot + * @param callback - user-defined function to format vslot + */ + int _mri_type_add_vslot(cstring_t type, cstring_t name, mri_formatter_cb callback) { + auto xtype_iter = mri_xtypes.find(type); + if (unlikely(mri_xtypes.end() == xtype_iter)) { + MRI_LOG(MRI_ERROR, "xtype %s does not exist, better luck next time", type); + return -EINVAL; + } + + /* TODO(omer): define this somewhere and do something about snprintf-return */ + char virt_formatter[256]; + (void) snprintf(virt_formatter, sizeof(virt_formatter), "__%s_%s", type, name); + + /* Creates 2 variables: iter, did_insert (emplace result) */ + MRI_MAP_INSERT(mri_formatters, virt_formatter, callback); + if (unlikely(! did_insert)) { + if (mri_formatters.end() == iter) { + MRI_LOG(MRI_ERROR, "Failed to insert virtual-format for %s::%s (OOM)", type, name); + return -ENOMEM; + } + + MRI_LOG(MRI_ERROR, "Seems like you already configured vslot (%s::%s), naughty developer ..", type, name); + return -EEXIST; + } + + auto const &format = iter->second; + auto &mri_type = xtype_iter->second; + return mri_type.add_slot(name, 0, mri_type.m_size, format.m_formatter, 0); + } + + int _mri_type_add_shaper( cstring_t type, + cstring_t name, + cstring_t shaper, + size_t offset, + size_t size) { + auto xtype_iter = mri_xtypes.find(type); + if (unlikely(mri_xtypes.end() == xtype_iter)) { + MRI_LOG(MRI_ERROR, "xtype %s does not exist, better luck next time", type); + return -EINVAL; + } + + auto shaper_iter = mri_shapers.find(shaper); + if (mri_shapers.end() == shaper_iter) { + MRI_LOG(MRI_ERROR, "shaper %s does not exist, better luck next time", shaper); + return -EINVAL; + } + + auto &mri_type = xtype_iter->second; + + /* Add hidden slot for shaping (allow for user to add it as not-hidden before) */ + int retval = mri_type.add_slot(name, offset, size, nullptr, MRI_SLOT_FLAG_HIDDEN); + if (unlikely(0 != retval && -EEXIST != retval)) return retval; + + /* Save shaper to use later in xdump */ + auto const &xshaper = shaper_iter->second; + auto xslot_iter = mri_type.m_slots.find(name); + xslot_iter->second.m_shaper = xshaper.m_shaper; + + return 0; + } + + /* --------------------------------------------------------------------------------------------- + * Event-loop and timing related functions + * --------------------------------------------------------------------------------------------- + */ + + int mri_get_eventloop_fd(); + int mri_get_notification_fd(); + + /* --------------------------------------------------------------------------------------------- + * Custom user-data registration related functions + * --------------------------------------------------------------------------------------------- + */ + + int _mri_register(cstring_t path, cstring_t type, void *object, mri_iterator_cb callback); + int mri_register_config(cstring_t path, void *context, mri_config_change_cb callback); + int mri_unregister(cstring_t path); +} From f5f76a64995dc820eb0ee0cc442f380a0b0ba694 Mon Sep 17 00:00:00 2001 From: Omer Shamash Date: Mon, 16 Mar 2020 17:39:08 +0000 Subject: [PATCH 4/7] MRI: v1: UMake IMake WeMake it work --- UMakefile | 20 +++++ include/mri.h | 61 +------------- src/mri_backend.cpp | 196 ++++++-------------------------------------- 3 files changed, 51 insertions(+), 226 deletions(-) create mode 100644 UMakefile diff --git a/UMakefile b/UMakefile new file mode 100644 index 0000000..e4e4bbd --- /dev/null +++ b/UMakefile @@ -0,0 +1,20 @@ +$nil = + +$cflags = -O2 -Wall -Wextra -Werror -pedantic -fPIC +$lflags = -Wl,-zdefs -fPIC + +[variant:debug] +$cflags = -O0 -g -ggdb -Wall -Wextra -Werror -pedantic -fPIC +$lflags = -Wl,-zdefs -fPIC + +!c(includes, flags=$nil, cflags=$cflags) : gcc -c {filename} $includes $cflags $flags -o {target} > {dir}/{noext}.umake.o +!cpp(includes, flags=$nil, cflags=$cflags) : g++ -std=c++17 -c {filename} $includes $cflags $flags -o {target} > {dir}/{noext}.umake.o +!so(libs, flags=$nil, lflags=$lflags) : g++ -std=c++17 --shared {filename} $lflags $flags $libs -o {target} +!a(libs) : ar rcs {target} {filename} $libs + +$mri_includes = -Iinclude -Ideps +$mri_cxx_flags = -fno-exceptions + +:foreach src/*.c > !c($mri_includes) +:foreach src/*.cpp > !cpp($mri_includes, $mri_cxx_flags) +: src/*.umake.o > !so( ) > libmri.so diff --git a/include/mri.h b/include/mri.h index 39e0be5..503fd79 100644 --- a/include/mri.h +++ b/include/mri.h @@ -1,66 +1,12 @@ #ifndef _MRI_H_ #define _MRI_H_ -#include -#include - /* Logger related */ +#include "mri_typing.h" #include "mri_logging.h" -/* defines */ -#define MRI_MAX_SLOT_STR_SIZE (256) -#define MRI_USERDATA_SIZE (32) - -#define MRI_SLOT_FLAG_CONST (1 << 0) /* TODO(omer): Unused for now */ -#define MRI_SLOT_FLAG_RATE (1 << 1) /* TODO(omer): Unused for now */ -#define MRI_SLOT_FLAG_PK (1 << 2) -#define MRI_SLOT_FLAG_HIDDEN (1 << 3) - -typedef enum mri_iter_action { - MRI_ITER_STOP, - MRI_ITER_CONTINUE -} mri_iter_action_t; - -/* typedefs */ -typedef uint8_t mri_byte_t; -typedef const char *cstring_t; - -typedef struct mri_iter_state { - size_t iteration_count; /* Iteration count (incremented by MRI) */ - mri_byte_t userdata[MRI_USERDATA_SIZE]; /* User provided data to help iteration */ -} mri_iter_state_t; - -/* Reflects result of capturing a slot */ -typedef struct mri_capture_sample { - uint32_t ns_timestamp; /* Data timestamp (monotonic nanoseconds) */ - mri_byte_t sample[MRI_MAX_SLOT_STR_SIZE]; /* Data captured via calls to the formatter */ -} mri_capture_sample_t; - -/* Provided formatters (multi-formatters) */ -typedef struct {} mri_hex_t; /* Hexadecimal representation */ -typedef struct {} mri_number_t; /* Plain normal numeric data */ -typedef struct {} mri_buffer_t; /* Memory address dump in bytes */ -typedef struct {} mri_string_t; /* ASCII encoded human string */ -typedef struct {} mri_timespec_t; /* timespec (tv_sec, tv_usec) */ -typedef struct {} mri_epoch_time_t; /* uint64_t seconds since epoch */ -typedef struct {} mri_elapsed_time_t; /* uint64_t monotonic diff */ - -/* Provided shapers (multi-shapers) */ -typedef struct {} mri_rate_shaper_t; /* Numeric slot rate per second */ - -/* Callbacks (TBD) */ - -/* Input: slot-data, size-of-slot, output-string */ -typedef int (*mri_formatter_cb)(void *, size_t, char *); -/* Input: type-data, iteration-state-data, output-memory */ -typedef mri_iter_action_t (*mri_iterator_cb)(void *, mri_iter_state_t *, void **); -/* Input: registered-context, cli-input, size-of-input */ -typedef int (*mri_config_change_cb)(void *, cstring_t, size_t); -/* Input: list-of-samples, amount-of-samples, output-string */ -typedef int (*mri_shaper_cb)(mri_capture_sample_t *, size_t, char *); - /* prototypes */ - + /** * mri_init() * @@ -69,9 +15,10 @@ typedef int (*mri_shaper_cb)(mri_capture_sample_t *, size_t, char *); * Instance is global to process having thread-oriented CB support * * @param domain - root-path, shared, kind of like catagory (routing, system-events, em, etc...) + * @param mode - mode to use MRI as (for service-discovery) * @returns C-Style boolean (0 = OK, <0 = ERROR) (-ERRNO) */ -int mri_init(cstring_t domain); +int mri_init(cstring_t domain, mri_service_mode_t mode); /* Formatting apart from provided-formatters section */ int _mri_create_formatter(cstring_t name, mri_formatter_cb callback); diff --git a/src/mri_backend.cpp b/src/mri_backend.cpp index deeb283..fb7d37b 100644 --- a/src/mri_backend.cpp +++ b/src/mri_backend.cpp @@ -1,166 +1,19 @@ #include #include +#include #include #include /* TODO(omer): Fix include paths */ extern "C" { - #include "../include/mri.h" + #include "mri.h" // #include "dn_common/general.h" } -#include "general.hpp" +#include "mri_general.hpp" //#include "../deps/ordered_map.h" -/* TODO(omer): Change to something real */ -#define mri_assert(cond) (void) 0 - -/* gettid() is undefined in glibc, use this instead */ -#include -#include -typedef pid_t tid_t; -static tid_t _gettid() { return syscall(SYS_gettid); } - -/* Declarations (should move soon) */ -struct xtype; -struct xformat; -struct xshaper; -struct mri_slot; -struct xpath_node; - -using mri_slot_map_t = std::map; -using mri_subpath_map_t = std::map; -using mri_capture_map_t = std::unordered_map>; -using mri_formatter_map_t = std::unordered_map; -using mri_shaper_map_t = std::unordered_map; -using mri_xtype_map_t = std::unordered_map; - -struct xformat { - std::string m_name; - mri_formatter_cb m_formatter; - - template - xformat(string_t &&name, mri_formatter_cb formatter) - : m_name(std::forward(name)) - , m_formatter(formatter) - {} -}; - -struct xshaper { - std::string m_name; - mri_shaper_cb m_shaper; - - template - xshaper(string_t &&name, mri_shaper_cb shaper) - : m_name(std::forward(name)) - , m_shaper(shaper) - {} -}; - -/** - * MRI_SLOT - * Member of a registered user-defined data structure - */ -struct mri_slot { - std::string m_name; /*!< Name of the slot (unique per-xtype) */ - size_t m_offset; /*!< Offset of the slot's data in xtype */ - size_t m_size; /*!< Size of the slot (used by formatter) */ - mri_formatter_cb m_formatter; /*!< Reference to formatter (int, str, ...) */ - mri_shaper_cb m_shaper; /*!< Pointer to slot data-samples shaper_cb */ - bool m_is_hidden; /*!< Indicator for external usage (to-show) */ - - /* TODO(omer): Find a better way (m_shaper doubles as is-slot-shaper indicator) */ - /* Note: m_shaper will be filled externally for xpath_node::xtype::dump to use */ - template - mri_slot( string_t &&name, - size_t offset, - size_t size, - mri_formatter_cb formatter, - bool hidden) - : m_name(std::forward(name)) - , m_offset(offset) - , m_size(size) - , m_formatter(formatter) - , m_shaper(nullptr) - , m_is_hidden(hidden) - {} -}; - -/* TODO(omer): Consider enabling a flat-parse for an xtype (slot, slot, slot, ...) */ - -/** - * XTYPE - * Type which is composed from multiple slots (complex type) - * Those types are usually being registered to one or more xpath_node - */ -struct xtype { - std::string m_name; /*!< Name of this X-Type (unique) */ - size_t m_size; /*!< Size needed to allocate the type */ - mri_slot_map_t m_slots; /*!< Ordered mapping of all type's slots */ - struct mri_slot *m_private_key; /*!< Explicit pointer to the type's PK slot */ - mri_capture_map_t m_slot_capture_map; /*!< Externalize the per-slot data captures */ - - template - xtype(string_t &&name, size_t size) - : m_name(std::forward(name)) - , m_size(size) - , m_slots() - , m_private_key(nullptr) - , m_slot_capture_map() - {} - - /** - * Create a slot (data-member) - * @param name - name of the new slot - * @param offset - byte offset in data struct (offsetof) - * @param size - size in bytes of contained data (sizeof) - * @param formatter - format callback for the new slot (to_string) - * @param flags - bitwise flags of slot options (pk, rate, hidden, etc...) - * @return C-Style boolean (0 = OK, <0 = ERROR) (-ERRNO) - */ - int add_slot(cstring_t name, size_t offset, size_t size, mri_formatter_cb formatter, int flags); -}; - -/** - * xpath_node - * Chains of nodes defining path (/ is root) - */ -struct xpath_node { - /* General definition of a node */ - std::string m_name; /*>! Name of node (name unique per parent) */ - mri_subpath_map_t m_sub_nodes; /*>! Mapping of sub-nodes (each is unique) */ - xtype *m_type; /*>! Type coresponding with the node's data */ - xpath_node *m_parent; /*>! Reverse reference to the parent's node */ - - /* User-data (specified on registration) */ - tid_t m_tid; /*>! Thread whom registered node (origin) */ - void *m_xdata; /*>! User-defined data (node data internal) */ - mri_iterator_cb m_xdata_iterator; /*>! User-defined data iterator (tableview) */ - - /** - * Notes: - * 1. No type = empty node, path-building - * 2. No parent = root node, others should not do this - * 3. No data = only allowed for empty node - * 4. Iterator = optional, data can be flat - * 5. Name cannot contain / (except root) to allow path traversal - * 6. Path is defined as ///.../ - */ - template - xpath_node( string_t &&name, - xtype *type, - xpath_node *parent, - void *xdata, - mri_iterator_cb data_iter) - : m_name(std::forward(name)) - , m_sub_nodes() - , m_type(type) - , m_parent(parent) - , m_tid(_gettid()) - , m_xdata(xdata) - , m_xdata_iterator(data_iter) - { mri_assert(parent || ! data); } -}; +#include "mri_data.hpp" /* Global variables */ static mri_xtype_map_t mri_xtypes; @@ -172,11 +25,9 @@ static xpath_node mri_root ("/", nullptr, nullptr, nullptr, nullptr); /* Note: ignore potential copy, even by using std::move, compiler optimization will fix this */ \ /* Note: usage of m_name.c_str() as the map-key allows us to avoid another string allocation */ \ decltype(map)::mapped_type __temp_cls (key, ##__VA_ARGS__); \ - auto&& [iter, did_insert] = map.try_emplace(__temp_cls.m_name.c_str(), std::move(__temp_cls)) \ + auto&& [iter, did_insert] = map.try_emplace(__temp_cls.m_name.c_str(), std::move(__temp_cls)) -#define MRI_SAFE_MAP_INSERTION(exists_err, map_data_t, map, key, ...) \ - MRI_MAP_INSERT(map, key, ##__VA_ARGS__); \ - \ +#define MRI_MAP_INSERT_CHECK_RES(exists_err, map_data_t, map, key) \ if (likely(did_insert)) return 0; /* Bailout if all worked */ \ \ if (map.end() == iter) { \ @@ -187,14 +38,21 @@ static xpath_node mri_root ("/", nullptr, nullptr, nullptr, nullptr); MRI_LOG(MRI_WARNING, #map_data_t " %s already exists", key); \ return exists_err -int xtype::add_slot(cstring_t name, size_t offset, size_t size, mri_formatter_cb formatter, int flags) { - (void) name; - (void) offset; - (void) size; - (void) formatter; - (void) flags; - - return 0; +#define MRI_SAFE_MAP_INSERTION(exists_err, map_data_t, map, key, ...) \ + MRI_MAP_INSERT(map, key, ##__VA_ARGS__); \ + MRI_MAP_INSERT_CHECK_RES(exists_err, map_data_t, map, key) + +/* Easier to implement here due to macros, consider moving */ +int xtype::add_slot(cstring_t slot, size_t offset, size_t size, mri_formatter_cb formatter, int flags) { + MRI_SAFE_MAP_INSERTION( -EEXIST, + mri_slot, + m_slots, + /* mri_slot construction */ + slot, + offset, + size, + formatter, + (flags & MRI_SLOT_FLAG_HIDDEN)); } extern "C" { @@ -305,7 +163,7 @@ extern "C" { } int _mri_type_add_shaper( cstring_t type, - cstring_t name, + cstring_t slot, cstring_t shaper, size_t offset, size_t size) { @@ -324,15 +182,15 @@ extern "C" { auto &mri_type = xtype_iter->second; /* Add hidden slot for shaping (allow for user to add it as not-hidden before) */ - int retval = mri_type.add_slot(name, offset, size, nullptr, MRI_SLOT_FLAG_HIDDEN); + int retval = mri_type.add_slot(slot, offset, size, nullptr, MRI_SLOT_FLAG_HIDDEN); if (unlikely(0 != retval && -EEXIST != retval)) return retval; /* Save shaper to use later in xdump */ auto const &xshaper = shaper_iter->second; - auto xslot_iter = mri_type.m_slots.find(name); - xslot_iter->second.m_shaper = xshaper.m_shaper; - - return 0; + auto &xslot = mri_type.m_slots.find(slot)->second; + xslot.m_shaper = xshaper.m_shaper; + auto&& [iter, did_insert] = mri_type.m_slot_capture_map.try_emplace(slot, slot_capture_data {}); + MRI_MAP_INSERT_CHECK_RES(-EEXIST, slot_capture_data, mri_type.m_slot_capture_map, slot); } /* --------------------------------------------------------------------------------------------- From 2046dd2c9bf76f4bc3577b29dbd68496b3ef59e2 Mon Sep 17 00:00:00 2001 From: Omer Shamash Date: Sat, 21 Mar 2020 18:30:00 +0000 Subject: [PATCH 5/7] MR: v1: Add initial files from last sprint --- .ycm_extra_conf.py | 222 ++++++++++++++++++++++++++++++++++ include/mri_logging.h | 27 +++++ include/mri_typing.h | 65 ++++++++++ src/mri_config.c | 26 ++++ src/mri_config.h | 58 +++++++++ src/mri_data.hpp | 165 +++++++++++++++++++++++++ src/mri_general.h | 19 +++ src/mri_general.hpp | 18 +++ src/mri_logger.c | 39 ++++++ src/mri_service_discovery.hpp | 34 ++++++ src/mri_typing.hpp | 19 +++ 11 files changed, 692 insertions(+) create mode 100644 .ycm_extra_conf.py create mode 100644 include/mri_logging.h create mode 100644 include/mri_typing.h create mode 100644 src/mri_config.c create mode 100644 src/mri_config.h create mode 100644 src/mri_data.hpp create mode 100644 src/mri_general.h create mode 100644 src/mri_general.hpp create mode 100644 src/mri_logger.c create mode 100644 src/mri_service_discovery.hpp create mode 100644 src/mri_typing.hpp diff --git a/.ycm_extra_conf.py b/.ycm_extra_conf.py new file mode 100644 index 0000000..bebc38b --- /dev/null +++ b/.ycm_extra_conf.py @@ -0,0 +1,222 @@ +# This file is NOT licensed under the GPLv3, which is the license for the rest +# of YouCompleteMe. +# +# Here's the license text for this file: +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# For more information, please refer to + +from distutils.sysconfig import get_python_inc +import os +import platform +import os.path as p +import subprocess + +DIR_OF_THIS_SCRIPT = p.abspath( p.dirname( __file__ ) ) +DIR_OF_THIRD_PARTY = p.join( DIR_OF_THIS_SCRIPT, 'third_party' ) +SOURCE_EXTENSIONS = [ '.cpp', '.cxx', '.cc', '.c', '.m', '.mm' ] + +database = None + +# These are the compilation flags that will be used in case there's no +# compilation database set (by default, one is not set). +# CHANGE THIS LIST OF FLAGS. YES, THIS IS THE DROID YOU HAVE BEEN LOOKING FOR. +flags = [ +'-Wall', +'-Wextra', +'-Werror', +'-Wno-long-long', +'-Wno-variadic-macros', +'-fexceptions', +'-DNDEBUG', +# You 100% do NOT need -DUSE_CLANG_COMPLETER and/or -DYCM_EXPORT in your flags; +# only the YCM source code needs it. +'-DUSE_CLANG_COMPLETER', +'-DYCM_EXPORT=', +# Custome flags +'-Iinclude', +# THIS IS IMPORTANT! Without the '-x' flag, Clang won't know which language to +# use when compiling headers. So it will guess. Badly. So C++ headers will be +# compiled as C headers. You don't want that so ALWAYS specify the '-x' flag. +# For a C project, you would set this to 'c' instead of 'c++'. +'-x', +'c++', +'-isystem', +'cpp/pybind11', +'-isystem', +'cpp/whereami', +'-isystem', +'cpp/BoostParts', +'-isystem', +get_python_inc(), +'-isystem', +'cpp/llvm/include', +'-isystem', +'cpp/llvm/tools/clang/include', +'-I', +'cpp/ycm', +'-I', +'cpp/ycm/ClangCompleter', +'-isystem', +'cpp/ycm/tests/gmock/gtest', +'-isystem', +'cpp/ycm/tests/gmock/gtest/include', +'-isystem', +'cpp/ycm/tests/gmock', +'-isystem', +'cpp/ycm/tests/gmock/include', +'-isystem', +'cpp/ycm/benchmarks/benchmark/include', +] + +# Clang automatically sets the '-std=' flag to 'c++14' for MSVC 2015 or later, +# which is required for compiling the standard library, and to 'c++11' for older +# versions. +if platform.system() != 'Windows': + flags.append( '-std=c++17' ) + + +# Set this to the absolute path to the folder (NOT the file!) containing the +# compile_commands.json file to use that instead of 'flags'. See here for +# more details: http://clang.llvm.org/docs/JSONCompilationDatabase.html +# +# You can get CMake to generate this file for you by adding: +# set( CMAKE_EXPORT_COMPILE_COMMANDS 1 ) +# to your CMakeLists.txt file. +# +# Most projects will NOT need to set this to anything; you can just change the +# 'flags' list of compilation flags. Notice that YCM itself uses that approach. +compilation_database_folder = '' + + +def IsHeaderFile( filename ): + extension = p.splitext( filename )[ 1 ] + return extension in [ '.h', '.hxx', '.hpp', '.hh' ] + + +def FindCorrespondingSourceFile( filename ): + if IsHeaderFile( filename ): + basename = p.splitext( filename )[ 0 ] + for extension in SOURCE_EXTENSIONS: + replacement_file = basename + extension + if p.exists( replacement_file ): + return replacement_file + return filename + + +def PathToPythonUsedDuringBuild(): + try: + filepath = p.join( DIR_OF_THIS_SCRIPT, 'PYTHON_USED_DURING_BUILDING' ) + with open( filepath ) as f: + return f.read().strip() + except OSError: + return None + + +def Settings( **kwargs ): + # Do NOT import ycm_core at module scope. + import ycm_core + + global database + if database is None and p.exists( compilation_database_folder ): + database = ycm_core.CompilationDatabase( compilation_database_folder ) + + language = kwargs[ 'language' ] + + if language == 'cfamily': + # If the file is a header, try to find the corresponding source file and + # retrieve its flags from the compilation database if using one. This is + # necessary since compilation databases don't have entries for header files. + # In addition, use this source file as the translation unit. This makes it + # possible to jump from a declaration in the header file to its definition + # in the corresponding source file. + filename = FindCorrespondingSourceFile( kwargs[ 'filename' ] ) + + if not database: + return { + 'flags': flags, + 'include_paths_relative_to_dir': DIR_OF_THIS_SCRIPT, + 'override_filename': filename + } + + compilation_info = database.GetCompilationInfoForFile( filename ) + if not compilation_info.compiler_flags_: + return {} + + # Bear in mind that compilation_info.compiler_flags_ does NOT return a + # python list, but a "list-like" StringVec object. + final_flags = list( compilation_info.compiler_flags_ ) + + # NOTE: This is just for YouCompleteMe; it's highly likely that your project + # does NOT need to remove the stdlib flag. DO NOT USE THIS IN YOUR + # ycm_extra_conf IF YOU'RE NOT 100% SURE YOU NEED IT. + try: + final_flags.remove( '-stdlib=libc++' ) + except ValueError: + pass + + return { + 'flags': final_flags, + 'include_paths_relative_to_dir': compilation_info.compiler_working_dir_, + 'override_filename': filename + } + + if language == 'python': + return { + 'interpreter_path': PathToPythonUsedDuringBuild() + } + + return {} + + +def PythonSysPath( **kwargs ): + sys_path = kwargs[ 'sys_path' ] + + interpreter_path = kwargs[ 'interpreter_path' ] + major_version = subprocess.check_output( [ + interpreter_path, '-c', 'import sys; print( sys.version_info[ 0 ] )' ] + ).rstrip().decode( 'utf8' ) + + sys_path[ 0:0 ] = [ p.join( DIR_OF_THIS_SCRIPT ), + p.join( DIR_OF_THIRD_PARTY, 'bottle' ), + p.join( DIR_OF_THIRD_PARTY, 'cregex', + 'regex_{}'.format( major_version ) ), + p.join( DIR_OF_THIRD_PARTY, 'frozendict' ), + p.join( DIR_OF_THIRD_PARTY, 'jedi_deps', 'jedi' ), + p.join( DIR_OF_THIRD_PARTY, 'jedi_deps', 'parso' ), + p.join( DIR_OF_THIRD_PARTY, 'requests_deps', 'requests' ), + p.join( DIR_OF_THIRD_PARTY, 'requests_deps', + 'urllib3', + 'src' ), + p.join( DIR_OF_THIRD_PARTY, 'requests_deps', + 'chardet' ), + p.join( DIR_OF_THIRD_PARTY, 'requests_deps', + 'certifi' ), + p.join( DIR_OF_THIRD_PARTY, 'requests_deps', + 'idna' ), + p.join( DIR_OF_THIRD_PARTY, 'waitress' ) ] + + sys_path.append( p.join( DIR_OF_THIRD_PARTY, 'jedi_deps', 'numpydoc' ) ) + return sys_path diff --git a/include/mri_logging.h b/include/mri_logging.h new file mode 100644 index 0000000..769986b --- /dev/null +++ b/include/mri_logging.h @@ -0,0 +1,27 @@ +#ifndef _MRI_LOGGING_H +#define _MRI_LOGGING_H + +#include + +/* Severity similar to syslog */ +typedef enum mri_severity { + MRI_DEBUG = 7, + MRI_NOTICE = 6, + MRI_INFO = 5, + MRI_WARNING = 4, + #define MRI_WARN MRI_WARNING + MRI_ERROR = 3, + #define MRI_ERR MRI_ERROR + MRI_CRITICAL = 2, + #define MRI_CRIT MRI_CRITICAL + MRI_ALERT = 1, + MRI_EMERG = 0 + #define MRI_PANIC MRI_EMERG +} mri_severity_t; + +/* Note: it is sufficient to override just one func */ +/* Weak-ref logging functions defaulted to printf() */ +extern void mri_log(mri_severity_t, const char *, ...); +extern void mri_vlog(mri_severity_t, const char *, va_list); + +#endif diff --git a/include/mri_typing.h b/include/mri_typing.h new file mode 100644 index 0000000..f6523f7 --- /dev/null +++ b/include/mri_typing.h @@ -0,0 +1,65 @@ +#ifndef _MRI_TYPING_H_ +#define _MRI_TYPING_H_ + +#include +#include + +/* defines */ +#define MRI_MAX_HOST_SIZE (128) +#define MRI_MAX_SLOT_STR_SIZE (256) +#define MRI_USERDATA_SIZE (32) + +#define MRI_SLOT_FLAG_CONST (1 << 0) /* TODO(omer): Unused for now */ +#define MRI_SLOT_FLAG_RATE (1 << 1) /* TODO(omer): Unused for now */ +#define MRI_SLOT_FLAG_PK (1 << 2) +#define MRI_SLOT_FLAG_HIDDEN (1 << 3) + +typedef enum mri_service_mode { + MRI_PRODUCER = 0b01, + MRI_CONSUMER = 0b10 +} mri_service_mode_t; + +typedef enum mri_iter_action { + MRI_ITER_STOP, + MRI_ITER_CONTINUE +} mri_iter_action_t; + +/* typedefs */ +typedef uint8_t mri_byte_t; +typedef const char *cstring_t; + +typedef struct mri_iter_state { + size_t iteration_count; /* Iteration count (incremented by MRI) */ + mri_byte_t userdata[MRI_USERDATA_SIZE]; /* User provided data to help iteration */ +} mri_iter_state_t; + +/* Reflects result of capturing a slot */ +typedef struct mri_capture_sample { + uint32_t ns_timestamp; /* Data timestamp (monotonic nanoseconds) */ + mri_byte_t sample[MRI_MAX_SLOT_STR_SIZE]; /* Data captured via calls to the formatter */ +} mri_capture_sample_t; + +/* Provided formatters (multi-formatters) */ +typedef struct {} mri_hex_t; /* Hexadecimal representation */ +typedef struct {} mri_number_t; /* Plain normal numeric data */ +typedef struct {} mri_buffer_t; /* Memory address dump in bytes */ +typedef struct {} mri_string_t; /* ASCII encoded human string */ +typedef struct {} mri_timespec_t; /* timespec (tv_sec, tv_usec) */ +typedef struct {} mri_epoch_time_t; /* uint64_t seconds since epoch */ +typedef struct {} mri_elapsed_time_t; /* uint64_t monotonic diff */ + +/* Provided shapers (multi-shapers) */ +typedef struct {} mri_rate_shaper_t; /* Numeric slot rate per second */ + +/* Callbacks (TBD) */ + +/* Input: slot-data, size-of-slot, output-string */ +typedef int (*mri_formatter_cb)(void *, size_t, char *); +/* Input: type-data, iteration-state-data, output-memory */ +typedef mri_iter_action_t (*mri_iterator_cb)(void *, mri_iter_state_t *, void **); +/* Input: registered-context, cli-input, size-of-input */ +typedef int (*mri_config_change_cb)(void *, cstring_t, size_t); +/* Input: list-of-samples, amount-of-samples, output-string */ +typedef int (*mri_shaper_cb)(mri_capture_sample_t *, size_t, char *); + +#endif diff --git a/src/mri_config.c b/src/mri_config.c new file mode 100644 index 0000000..13a681c --- /dev/null +++ b/src/mri_config.c @@ -0,0 +1,26 @@ +#include "mri_config.h" +#include "mri_general.h" + +#include +#include + +typedef struct mri_config_data { + void *data; + size_t length; +} mri_config_data_t; + + +struct mri_configuration { + mri_config_data_t config[MRI_CONFIG_MAX_VALUE]; +}; + +struct mri_configuration *mri_config_create(int argc, char **argv) { + struct mri_configuration *conf = + (struct mri_configuration *) calloc(1, sizeof(struct mri_configuration)); + if (NULL == conf) { + MRI_LOG(MRI_ERROR, "Failed to allocate memory for configuration (OOM)"); + return NULL; + } + + return conf; +} diff --git a/src/mri_config.h b/src/mri_config.h new file mode 100644 index 0000000..dfd2cce --- /dev/null +++ b/src/mri_config.h @@ -0,0 +1,58 @@ +#ifndef _MRI_CONFIG_H_ +#define _MRI_CONFIG_H_ + +#include "mri_typing.h" + +/* FWD Decleration */ +struct mri_configuration; + +enum mri_config_value { + /* Producer configuration */ + MRI_CONFIG_DOMAIN, + + /* Service discovery configuration */ + MRI_CONFIG_SERVICE_DISCOVERY_HOST, + MRI_CONFIG_SERVICE_DISCOVERY_PORT, + MRI_CONFIG_SERVICE_DISCOVERY_TO_USE, + + /* Never go beyond this point */ + MRI_CONFIG_MAX_VALUE +}; + +/** + * Create configuration object + * Input is formatted in getopt() fashion + * @param argc - main-like input, number of arguments + * @param argv - main-like inout, array of argument-strings + * @return mri_configuration on success, NULL on failure + */ +struct mri_configuration *mri_config_create(int argc, char **argv); + +/** + * Destroy configuration object + * @param config - mri configuration object to destroy + * @returns C-style boolean (0 = OK, <0 = ERROR) (-ERRNO) + */ +int mri_config_destroy(struct mri_configuration *config); + +/** + * Get configuration-value from object + * @param config - mri configuration object to manage + * @param type - type of configuration-value (enum) to manage + * @param output - output buffer/object to write value into + * @param size - size of output object, on success write-back size written + * @returns C-style boolean (0 = OK, <0 = ERROR) (-ERRNO) + */ +int mri_config_get(struct mri_configuration *config, mri_config_value type, void *output, size_t *size); + +/** + * Get configuration-value from object + * @param config - mri configuration object to manage + * @param type - type of configuration-value (enum) to manage + * @param input - input buffer/object to write as config-value + * @param size - size of input object + * @returns C-style boolean (0 = OK, <0 = ERROR) (-ERRNO) + */ +int mri_config_set(struct mri_configuration *config, mri_config_value type, void *input, size_t size); + +#endif diff --git a/src/mri_data.hpp b/src/mri_data.hpp new file mode 100644 index 0000000..8b97f51 --- /dev/null +++ b/src/mri_data.hpp @@ -0,0 +1,165 @@ +#ifndef _MRI_DATA_HPP_ +#define _MRI_DATA_HPP_ + +#include +#include +#include + +/* gettid() is undefined in glibc, use this instead */ +#include +#include +typedef pid_t tid_t; +static tid_t _gettid() { return syscall(SYS_gettid); } + +#include "mri_typing.hpp" + +/* TODO(omer): Fix this */ +#define mri_assert(cond) (void) 0 + +/* Declarations (should move soon) */ +struct xtype; +struct xformat; +struct xshaper; +struct mri_slot; +struct xpath_node; +struct slot_capture_data; + +using mri_slot_map_t = mri_ordered_map; +using mri_subpath_map_t = mri_ordered_map; +using mri_capture_map_t = mri_unordered_map; +using mri_formatter_map_t = mri_unordered_map; +using mri_shaper_map_t = mri_unordered_map; +using mri_xtype_map_t = mri_unordered_map; + +struct xformat { + std::string m_name; + mri_formatter_cb m_formatter; + + template + xformat(string_t &&name, mri_formatter_cb formatter) + : m_name(std::forward(name)) + , m_formatter(formatter) + {} +}; + +struct xshaper { + std::string m_name; + mri_shaper_cb m_shaper; + + template + xshaper(string_t &&name, mri_shaper_cb shaper) + : m_name(std::forward(name)) + , m_shaper(shaper) + {} +}; + +struct slot_capture_data { + mri_vector samples; +}; + +/** + * MRI_SLOT + * Member of a registered user-defined data structure + */ +struct mri_slot { + std::string m_name; /*!< Name of the slot (unique per-xtype) */ + size_t m_offset; /*!< Offset of the slot's data in xtype */ + size_t m_size; /*!< Size of the slot (used by formatter) */ + mri_formatter_cb m_formatter; /*!< Reference to formatter (int, str, ...) */ + mri_shaper_cb m_shaper; /*!< Pointer to slot data-samples shaper_cb */ + bool m_is_hidden; /*!< Indicator for external usage (to-show) */ + + /* TODO(omer): Find a better way (m_shaper doubles as is-slot-shaper indicator) */ + /* Note: m_shaper will be filled externally for xpath_node::xtype::dump to use */ + template + mri_slot( string_t &&name, + size_t offset, + size_t size, + mri_formatter_cb formatter, + bool hidden) + : m_name(std::forward(name)) + , m_offset(offset) + , m_size(size) + , m_formatter(formatter) + , m_shaper(nullptr) + , m_is_hidden(hidden) + {} +}; + +/* TODO(omer): Consider enabling a flat-parse for an xtype (slot, slot, slot, ...) */ + +/** + * XTYPE + * Type which is composed from multiple slots (complex type) + * Those types are usually being registered to one or more xpath_node + */ +struct xtype { + std::string m_name; /*!< Name of this X-Type (unique) */ + size_t m_size; /*!< Size needed to allocate the type */ + mri_slot_map_t m_slots; /*!< Ordered mapping of all type's slots */ + struct mri_slot *m_private_key; /*!< Explicit pointer to the type's PK slot */ + mri_capture_map_t m_slot_capture_map; /*!< Externalize the per-slot data captures */ + + template + xtype(string_t &&name, size_t size) + : m_name(std::forward(name)) + , m_size(size) + , m_slots() + , m_private_key(nullptr) + , m_slot_capture_map() + {} + + /** + * Create a slot (data-member) + * @param slot - name of the new slot + * @param offset - byte offset in data struct (offsetof) + * @param size - size in bytes of contained data (sizeof) + * @param formatter - format callback for the new slot (to_string) + * @param flags - bitwise flags of slot options (pk, rate, hidden, etc...) + * @return C-Style boolean (0 = OK, <0 = ERROR) (-ERRNO) + */ + int add_slot(cstring_t slot, size_t offset, size_t size, mri_formatter_cb formatter, int flags); +}; + +/** + * xpath_node + * Chains of nodes defining path (/ is root) + */ +struct xpath_node { + /* General definition of a node */ + std::string m_name; /*>! Name of node (name unique per parent) */ + mri_subpath_map_t m_sub_nodes; /*>! Mapping of sub-nodes (each is unique) */ + xtype *m_type; /*>! Type coresponding with the node's data */ + xpath_node *m_parent; /*>! Reverse reference to the parent's node */ + + /* User-data (specified on registration) */ + tid_t m_tid; /*>! Thread whom registered node (origin) */ + void *m_xdata; /*>! User-defined data (node data internal) */ + mri_iterator_cb m_xdata_iterator; /*>! User-defined data iterator (tableview) */ + + /** + * Notes: + * 1. No type = empty node, path-building + * 2. No parent = root node, others should not do this + * 3. No data = only allowed for empty node + * 4. Iterator = optional, data can be flat + * 5. Name cannot contain / (except root) to allow path traversal + * 6. Path is defined as ///.../ + */ + template + xpath_node( string_t &&name, + xtype *type, + xpath_node *parent, + void *xdata, + mri_iterator_cb data_iter) + : m_name(std::forward(name)) + , m_sub_nodes() + , m_type(type) + , m_parent(parent) + , m_tid(_gettid()) + , m_xdata(xdata) + , m_xdata_iterator(data_iter) + { mri_assert(parent || ! xdata); } +}; + +#endif diff --git a/src/mri_general.h b/src/mri_general.h new file mode 100644 index 0000000..498c377 --- /dev/null +++ b/src/mri_general.h @@ -0,0 +1,19 @@ +#ifndef _MRI_GENERAL_H_ +#define _MRI_GENERAL_H_ + +/* TODO(omer): Fix includes */ +#include "mri_logging.h" + +#ifndef likely + #define likely(x) __builtin_expect((x),1) +#endif +#ifndef unlikely + #define unlikely(x) __builtin_expect((x),0) +#endif + +#include +#include +extern char *__progname; +#define MRI_LOG(level, fmt, ...) \ + mri_log(level, "[%s:%d:%s] [%s:%d] " fmt, __FILE__, __LINE__, __PRETTY_FUNCTION__, __progname, (int) syscall(SYS_gettid), ##__VA_ARGS__) +#endif diff --git a/src/mri_general.hpp b/src/mri_general.hpp new file mode 100644 index 0000000..3df2de2 --- /dev/null +++ b/src/mri_general.hpp @@ -0,0 +1,18 @@ +#ifndef _MRI_GENERAL_HPP_ +#define _MRI_GENERAL_HPP_ + +#include "mri_general.h" + +template +struct c_return {}; + +template <> +struct c_return { + int value; + + constexpr c_return(bool val) + : value(val ? 0 : -1) + {} +}; + +#endif diff --git a/src/mri_logger.c b/src/mri_logger.c new file mode 100644 index 0000000..b25e2dd --- /dev/null +++ b/src/mri_logger.c @@ -0,0 +1,39 @@ +/* TODO(omer): Fix include paths */ +#include "../include/mri_logging.h" + +#include +#include + +const char *sev2string(mri_severity_t sev) { + switch (sev) { + case MRI_DEBUG : return "[DEBUG ]"; + case MRI_NOTICE : return "[NOTICE]"; + case MRI_INFO : return "[ INFO ]"; + case MRI_WARNING : return "[ WARN ]"; + case MRI_ERROR : return "[ERROR ]"; + case MRI_CRITICAL : return "[ CRIT ]"; + case MRI_ALERT : return "[ALERT ]"; + case MRI_EMERG : return "[EMERG ]"; + default : return "[BUGGED]"; + } +} + +/* Weak-ref logger functions */ + +void __attribute__((weak)) +mri_log(mri_severity_t level, const char *fmt, ...) { + va_list va; + va_start(va, fmt); + mri_vlog(level, fmt, va); + va_end(va); +} + +void __attribute__((weak)) +mri_vlog(mri_severity_t level, const char *fmt, va_list args) { + time_t time_now; + time(&time_now); + printf("%s %s ", ctime(&time_now), sev2string(level)); + vprintf(fmt, args); + printf("\n"); +} + diff --git a/src/mri_service_discovery.hpp b/src/mri_service_discovery.hpp new file mode 100644 index 0000000..8c0ad09 --- /dev/null +++ b/src/mri_service_discovery.hpp @@ -0,0 +1,34 @@ +#ifndef _MRI_SERVICE_DISCOVERY_HPP_ +#define _MRI_SERVICE_DISCOVERY_HPP_ + +extern "C" { + #include "mri_config.h" +} +#include "mri_typing.hpp" + +/* This will change everytime we change the mri_service_data structure */ +#define MRI_SERVICE_DISCOERY_PROTOCOL_VERSION 1 +struct mri_service_data { + uint32_t version; /*!< service data versioning first ! */ + char host[MRI_MAX_HOST_SIZE]; /*!< host containing the MRI instance */ + mri_service_mode_t mode; /*!< MRI usage mode (producer/consumer) */ +}; + +/* Data describing current service */ +extern mri_service_data self; + +using mri_service_map = mri_ordered_map; + +class IServiceDiscovery { +protected: + IServiceDiscovery() = default; +public: + IServiceDiscovery(IServiceDiscovery&) = delete; + IServiceDiscovery(IServiceDiscovery&&) = delete; + + /* Responsible add `self` if it decides that it is needed */ + virtual bool initialize(mri_configuration &mri_config) = 0; + virtual mri_service_map &fetch_services() = 0; +}; + +#endif diff --git a/src/mri_typing.hpp b/src/mri_typing.hpp new file mode 100644 index 0000000..65d1a32 --- /dev/null +++ b/src/mri_typing.hpp @@ -0,0 +1,19 @@ +#ifndef _MRI_TYPING_HPP_ +#define _MRI_TYPING_HPP_ + +#include "mri_typing.h" + +#include +#include +#include + +template +using mri_unordered_map = std::unordered_map; + +template +using mri_ordered_map = std::map; + +template +using mri_vector = std::vector; + +#endif From 06e23f894128fd91cbbe6303176991d83ca348f4 Mon Sep 17 00:00:00 2001 From: Omer Shamash Date: Sun, 22 Mar 2020 19:43:12 +0000 Subject: [PATCH 6/7] MRI: v1: Add configuration support ^^ --- UMakefile | 4 +- include/mri.h | 57 +-------- include/mri_all.h | 8 ++ {src => include}/mri_config.h | 36 +++++- include/mri_consumer.h | 6 + include/mri_producer.h | 55 ++++++++ include/mri_typing.h | 4 +- src/mri_config.c | 148 +++++++++++++++++++++- src/mri_config_ext.h | 20 +++ src/mri_data.hpp | 3 + src/mri_general.hpp | 4 +- src/{mri_backend.cpp => mri_producer.cpp} | 18 +-- src/mri_service_discovery_factory.cpp | 36 ++++++ 13 files changed, 315 insertions(+), 84 deletions(-) create mode 100644 include/mri_all.h rename {src => include}/mri_config.h (52%) create mode 100644 include/mri_consumer.h create mode 100644 include/mri_producer.h create mode 100644 src/mri_config_ext.h rename src/{mri_backend.cpp => mri_producer.cpp} (95%) create mode 100644 src/mri_service_discovery_factory.cpp diff --git a/UMakefile b/UMakefile index e4e4bbd..3f53483 100644 --- a/UMakefile +++ b/UMakefile @@ -1,10 +1,10 @@ $nil = -$cflags = -O2 -Wall -Wextra -Werror -pedantic -fPIC +$cflags = -O2 -Wall -Wextra -Werror -fPIC $lflags = -Wl,-zdefs -fPIC [variant:debug] -$cflags = -O0 -g -ggdb -Wall -Wextra -Werror -pedantic -fPIC +$cflags = -O0 -g -ggdb -Wall -Wextra -Werror -fPIC $lflags = -Wl,-zdefs -fPIC !c(includes, flags=$nil, cflags=$cflags) : gcc -c {filename} $includes $cflags $flags -o {target} > {dir}/{noext}.umake.o diff --git a/include/mri.h b/include/mri.h index 503fd79..98165f8 100644 --- a/include/mri.h +++ b/include/mri.h @@ -1,7 +1,7 @@ #ifndef _MRI_H_ #define _MRI_H_ -/* Logger related */ +#include "mri_config.h" #include "mri_typing.h" #include "mri_logging.h" @@ -14,59 +14,8 @@ * Each instance can only be linked to one domain (! consider !) * Instance is global to process having thread-oriented CB support * - * @param domain - root-path, shared, kind of like catagory (routing, system-events, em, etc...) - * @param mode - mode to use MRI as (for service-discovery) + * @param config - pointer to configuration object to use (ownership transfered) * @returns C-Style boolean (0 = OK, <0 = ERROR) (-ERRNO) */ -int mri_init(cstring_t domain, mri_service_mode_t mode); - -/* Formatting apart from provided-formatters section */ -int _mri_create_formatter(cstring_t name, mri_formatter_cb callback); - -/* TODO(omer): Add shaper configuration (amount of samples, scope and more */ - -/* Shaping apart from provided-shapers section */ -int _mri_create_shaper(cstring_t name, mri_shaper_cb callback); - -/* Registering types and data to MRI */ -int _mri_create_type(cstring_t, size_t); -int _mri_type_add_slot(cstring_t, cstring_t, cstring_t, size_t, size_t, int); -int _mri_type_add_vslot(cstring_t, cstring_t, mri_formatter_cb); -int _mri_type_add_shaper(cstring_t, cstring_t, cstring_t, size_t, size_t); - -/* Current suggestion: - * when someone registers data on path, we capture his thread-id and - * allow him to poll on an event object that is related to his thread-id - * - * mri_get_eventloop_fd() will allow user to allocate time for MRI in any bitch-thread - * mri_get_notification_fd() will allow to get the thread-bound event-fd to poll in relevant thread - */ -int mri_get_eventloop_fd(); -int mri_get_notification_fd(); - -/* Registering data of defined types to MRI */ -int _mri_register(cstring_t path, cstring_t type, void *object, mri_iterator_cb callback); -int mri_register_config(cstring_t path, void *context, mri_config_change_cb callback); -int mri_unregister(cstring_t path); - -/* Formalize API using macros */ -#define mri_create_type(c_type) \ - _mri_create_type(#c_type, sizeof(c_type)) -#define mri_type_add_slot(c_type, slot, format, flags) \ - _mri_type_add_slot(#c_type, #slot, #format, (size_t) &(((c_type *) 0)->slot), sizeof(((c_type *) 0)->slot), flags) -#define mri_type_add_vslot(c_type, slot, vslot_format) \ - _mri_type_add_vslot(#c_type, slot, vslot_foramt) -#define mri_register(path, c_type, object, callback) \ - _mri_register(path, #c_type, object, callback) -#define mri_create_formatter(c_type, callback) \ - _mri_create_fromatter(#c_type, callback) -#define mri_create_shaper(c_type, callback) \ - _mri_create_shaper(#c_type, callback) - -#define mri_type_add_shaper_slot(c_type, slot, format, shaper) \ - _mri_type_add_shaper(#c_type, #slot, #format, (size_t) &(((c_type *) 0)->slot), sizeof(((c_type *) 0)->slot), #shaper) - -#define mri_type_add_rate(c_type, slot) \ - mri_type_add_shaper(c_type, slot, mri_number_t, mri_rate_shaper_t) - +int mri_init(struct mri_coniguration **config); #endif /* _MRI_H_ */ diff --git a/include/mri_all.h b/include/mri_all.h new file mode 100644 index 0000000..80c005e --- /dev/null +++ b/include/mri_all.h @@ -0,0 +1,8 @@ +#ifndef _MRI_ALL_H_ +#define _MRI_ALL_H_ + +#include "mri.h" +#include "mri_consumer.h" +#include "mri_producer.h" + +#endif diff --git a/src/mri_config.h b/include/mri_config.h similarity index 52% rename from src/mri_config.h rename to include/mri_config.h index dfd2cce..c331bb5 100644 --- a/src/mri_config.h +++ b/include/mri_config.h @@ -3,10 +3,12 @@ #include "mri_typing.h" +#define MRI_CONFIG_MAX_STRING_SIZE 128 + /* FWD Decleration */ struct mri_configuration; -enum mri_config_value { +typedef enum mri_config_value { /* Producer configuration */ MRI_CONFIG_DOMAIN, @@ -17,7 +19,7 @@ enum mri_config_value { /* Never go beyond this point */ MRI_CONFIG_MAX_VALUE -}; +} mri_config_value_t; /** * Create configuration object @@ -36,23 +38,43 @@ struct mri_configuration *mri_config_create(int argc, char **argv); int mri_config_destroy(struct mri_configuration *config); /** - * Get configuration-value from object + * Get a mutable configuration-value from object (copy of internal) * @param config - mri configuration object to manage * @param type - type of configuration-value (enum) to manage * @param output - output buffer/object to write value into * @param size - size of output object, on success write-back size written * @returns C-style boolean (0 = OK, <0 = ERROR) (-ERRNO) */ -int mri_config_get(struct mri_configuration *config, mri_config_value type, void *output, size_t *size); +int mri_config_get_mutable(struct mri_configuration *config, mri_config_value_t type, void *output, size_t *size); + +/** + * Get a non-mutable configuration-value from object (reference to internal) + * @param config - mri configuration object to manage + * @param type - type of configuration-value (enum) to manage + * @param output - pointer to data-pointer which will point to internal-data after call + * @param size - pointer to size which will be assigned with internal-data size after call + * @returns C-style boolean (0 = OK, <0 = ERROR) (-ERRNO) + */ +int mri_config_get(struct mri_configuration *config, mri_config_value_t type, const void **output, size_t *size); + +/** + * Set a mutable configuration-value from object (copy to internal) + * @param config - mri configuration object to manage + * @param type - type of configuration-value (enum) to manage + * @param input - input buffer/object to write as config-value (deep-copy) + * @param size - size of input object + * @returns C-style boolean (0 = OK, <0 = ERROR) (-ERRNO) + */ +int mri_config_set_mutable(struct mri_configuration *config, mri_config_value_t type, void *input, size_t size); /** - * Get configuration-value from object + * Set a non-mutable configuration-value from object (internal reference to it) * @param config - mri configuration object to manage * @param type - type of configuration-value (enum) to manage - * @param input - input buffer/object to write as config-value + * @param input - buffer to set as config-value (pointer copy) * @param size - size of input object * @returns C-style boolean (0 = OK, <0 = ERROR) (-ERRNO) */ -int mri_config_set(struct mri_configuration *config, mri_config_value type, void *input, size_t size); +int mri_config_set(struct mri_configuration *config, mri_config_value_t type, void *input, size_t size); #endif diff --git a/include/mri_consumer.h b/include/mri_consumer.h new file mode 100644 index 0000000..3906863 --- /dev/null +++ b/include/mri_consumer.h @@ -0,0 +1,6 @@ +#ifndef _MRI_CONSUMER_H_ +#define _MRI_CONSUMER_H_ + +#include "mri_typing.h" + +#endif diff --git a/include/mri_producer.h b/include/mri_producer.h new file mode 100644 index 0000000..ddddf13 --- /dev/null +++ b/include/mri_producer.h @@ -0,0 +1,55 @@ +#ifndef _MRI_PRODUCER_H_ +#define _MRI_PRODUCER_H_ + +#include "mri_typing.h" + +/* Formatting apart from provided-formatters section */ +int _mri_create_formatter(cstring_t name, mri_formatter_cb callback); + +/* TODO(omer): Add shaper configuration (amount of samples, scope and more */ + +/* Shaping apart from provided-shapers section */ +int _mri_create_shaper(cstring_t name, mri_shaper_cb callback); + +/* Registering types and data to MRI */ +int _mri_create_type(cstring_t, size_t); +int _mri_type_add_slot(cstring_t, cstring_t, cstring_t, size_t, size_t, int); +int _mri_type_add_vslot(cstring_t, cstring_t, mri_formatter_cb); +int _mri_type_add_shaper(cstring_t, cstring_t, cstring_t, size_t, size_t); + +/* Current suggestion: + * when someone registers data on path, we capture his thread-id and + * allow him to poll on an event object that is related to his thread-id + * + * mri_get_eventloop_fd() will allow user to allocate time for MRI in any bitch-thread + * mri_get_notification_fd() will allow to get the thread-bound event-fd to poll in relevant thread + */ +int mri_get_eventloop_fd(); +int mri_get_notification_fd(); + +/* Registering data of defined types to MRI */ +int _mri_register(cstring_t path, cstring_t type, void *object, mri_iterator_cb callback); +int mri_register_config(cstring_t path, void *context, mri_config_change_cb callback); +int mri_unregister(cstring_t path); + +/* Formalize API using macros */ +#define mri_create_type(c_type) \ + _mri_create_type(#c_type, sizeof(c_type)) +#define mri_type_add_slot(c_type, slot, format, flags) \ + _mri_type_add_slot(#c_type, #slot, #format, (size_t) &(((c_type *) 0)->slot), sizeof(((c_type *) 0)->slot), flags) +#define mri_type_add_vslot(c_type, slot, vslot_format) \ + _mri_type_add_vslot(#c_type, slot, vslot_foramt) +#define mri_register(path, c_type, object, callback) \ + _mri_register(path, #c_type, object, callback) +#define mri_create_formatter(c_type, callback) \ + _mri_create_fromatter(#c_type, callback) +#define mri_create_shaper(c_type, callback) \ + _mri_create_shaper(#c_type, callback) + +#define mri_type_add_shaper_slot(c_type, slot, format, shaper) \ + _mri_type_add_shaper(#c_type, #slot, #format, (size_t) &(((c_type *) 0)->slot), sizeof(((c_type *) 0)->slot), #shaper) + +#define mri_type_add_rate(c_type, slot) \ + mri_type_add_shaper(c_type, slot, mri_number_t, mri_rate_shaper_t) + +#endif diff --git a/include/mri_typing.h b/include/mri_typing.h index f6523f7..494c8a8 100644 --- a/include/mri_typing.h +++ b/include/mri_typing.h @@ -15,8 +15,8 @@ #define MRI_SLOT_FLAG_HIDDEN (1 << 3) typedef enum mri_service_mode { - MRI_PRODUCER = 0b01, - MRI_CONSUMER = 0b10 + MRI_PRODUCER = (1 << 0), + MRI_CONSUMER = (1 << 1) } mri_service_mode_t; typedef enum mri_iter_action { diff --git a/src/mri_config.c b/src/mri_config.c index 13a681c..b0fd93b 100644 --- a/src/mri_config.c +++ b/src/mri_config.c @@ -3,17 +3,84 @@ #include #include +#include -typedef struct mri_config_data { - void *data; - size_t length; -} mri_config_data_t; +#include "mri_config_ext.h" +/* Note: remember to update to have proper debugging later */ +const char *config_type_to_string(mri_config_value_t type) { + switch(type) { + case MRI_CONFIG_DOMAIN : return "DOMAIN"; + case MRI_CONFIG_SERVICE_DISCOVERY_HOST : return "SERVICE_DISCOVERY_HOST"; + case MRI_CONFIG_SERVICE_DISCOVERY_PORT : return "SERVICE_DISCOVERY_PORT"; + case MRI_CONFIG_SERVICE_DISCOVERY_TO_USE: return "SERVICE_DISCOVERY_TO_USE"; + default : return "UNKNOWN"; + } +} struct mri_configuration { mri_config_data_t config[MRI_CONFIG_MAX_VALUE]; }; +int cstring_ref_handler(const char *input, mri_config_data_t *output) { + output->data = (void *) input; + output->length = strnlen(input, MRI_CONFIG_MAX_STRING_SIZE); + return 0; +} + +/* External handlers from project */ +extern int config_to_service_discovery(const char *, mri_config_data_t *); + +/* Array of typedef int (config_handler_cb)(const char *input, mri_config_data_t *output); */ +static int (*handle_config_cb[MRI_CONFIG_MAX_VALUE])(const char *, mri_config_data_t *) = { + /* Note: we assume input is from main() so no need to allocate */ + [MRI_CONFIG_DOMAIN] = cstring_ref_handler, + [MRI_CONFIG_SERVICE_DISCOVERY_HOST] = cstring_ref_handler, + [MRI_CONFIG_SERVICE_DISCOVERY_PORT] = cstring_ref_handler, + [MRI_CONFIG_SERVICE_DISCOVERY_TO_USE] = config_to_service_discovery +}; + +static int parse_opts(struct mri_configuration *conf, int argc, char **argv) { + int option_index; + struct option long_options[] = { + { "domain", required_argument, 0, MRI_CONFIG_DOMAIN }, + { "sd-host", required_argument, 0, MRI_CONFIG_SERVICE_DISCOVERY_HOST }, + { "sd-port", required_argument, 0, MRI_CONFIG_SERVICE_DISCOVERY_PORT }, + { "service-discovery", required_argument, 0, MRI_CONFIG_SERVICE_DISCOVERY_TO_USE }, + { 0, 0, 0, 0 } + }; + + /* Fetch first option and loop */ + int c = getopt_long(argc, argv, "", long_options, &option_index); + while (0 == c) { + struct option *curr_opt = &(long_options[option_index]); + MRI_LOG(MRI_DEBUG, "Parsing config-item %s", curr_opt->name); + + int config_item = curr_opt->val; + if (unlikely(config_item < 0 || config_item >= MRI_CONFIG_MAX_VALUE)) { + MRI_LOG(MRI_ERROR, "Config-item %s (%d) cannot be parsed (out-of-range)", curr_opt->name, config_item); + return -1; + } + + __auto_type callback = handle_config_cb[config_item]; + if (unlikely(! callback)) { + MRI_LOG(MRI_ERROR, "Config-item %s missing callback handler"); + return -1; + } + + if (0 != callback(optarg, &(conf->config[config_item]))) { + MRI_LOG(MRI_WARN, "Config-item %s failed parsing (value = %s) | ignoring", curr_opt->name, optarg); + } else { + MRI_LOG(MRI_DEBUG, "Config-item %s parsed successfully", curr_opt->name); + } + + /* Fetch next option */ + c = getopt_long(argc, argv, "", long_options, &option_index); + } + + return 0; +} + struct mri_configuration *mri_config_create(int argc, char **argv) { struct mri_configuration *conf = (struct mri_configuration *) calloc(1, sizeof(struct mri_configuration)); @@ -22,5 +89,78 @@ struct mri_configuration *mri_config_create(int argc, char **argv) { return NULL; } + /* Parse arguments into configuration */ + if (argc && 0 != parse_opts(conf, argc, argv)) { + mri_config_destroy(conf); + return NULL; + } + return conf; } + +int mri_config_destroy(struct mri_configuration *config) { + if (config) free(config); + return 0; +} + +int mri_config_get_mutable(struct mri_configuration *config, mri_config_value_t type, void *output, size_t *size) { + /* Sanity */ + if (! config || type < 0 || type > MRI_CONFIG_MAX_VALUE) return -1; + + mri_config_data_t *config_item = &(config->config[type]); + if (! config_item->data || ! config_item->length) { + MRI_LOG(MRI_ERROR, "Config-item (%s) has no proper data, aborting it", config_type_to_string(type)); + return -1; + } + + if (*size >= config_item->length) { + memcpy(output, config_item->data, config_item->length); + *size = config_item->length; + return 0; + } + + MRI_LOG(MRI_WARN, + "Config-item (%s) will truncate (need %zu bytes, got buffer with %zu bytes)", + config_type_to_string(type), + config_item->length, + *size); + return -1; +} + +int mri_config_get(struct mri_configuration *config, mri_config_value_t type, const void **output, size_t *size) { + /* Sanity */ + if (! config || type < 0 || type > MRI_CONFIG_MAX_VALUE) return -1; + + mri_config_data_t *config_item = &(config->config[type]); + *output = config_item->data; + *size = config_item->length; + return 0; +} + +int mri_config_set_mutable(struct mri_configuration *config, mri_config_value_t type, void *input, size_t size) { + /* Sanity */ + if (! config || type < 0 || type > MRI_CONFIG_MAX_VALUE || ! input || ! size) return -1; + + mri_config_data_t *config_item = &(config->config[type]); + + /* Allocate data */ + if (! (config_item->data = malloc(size))) { + MRI_LOG(MRI_ERROR, "Config-item (%s) cannot allocate %zu bytes (OOM)", config_type_to_string(type), size); + return -1; + } + + memcpy(config_item->data, input, size); + config_item->length = size; + return 0; +} + + +int mri_config_set(struct mri_configuration *config, mri_config_value_t type, void *input, size_t size) { + /* Sanity */ + if (! config || type < 0 || type > MRI_CONFIG_MAX_VALUE) return -1; + + mri_config_data_t *config_item = &(config->config[type]); + config_item->data = input; + config_item->length = size; + return 0; +} diff --git a/src/mri_config_ext.h b/src/mri_config_ext.h new file mode 100644 index 0000000..b620355 --- /dev/null +++ b/src/mri_config_ext.h @@ -0,0 +1,20 @@ +#ifndef _MRI_CONFIG_EXT_H_ +#define _MRI_CONFIG_EXT_H_ + +#include + +/* Note: data is never free()'d - stays as long as the program does */ +typedef struct mri_config_data { + void *data; /*!< config-item data buffer (allocate if needed) */ + size_t length; /*!< Length of buffer (if dynamic sizing is needed) */ +} mri_config_data_t; + +/** + * Signature for config handlers (defined as externals inside mri_config.c) + * @param input - input string -> used-data from main() + * @param output - where to save configuration after parsing + * @return C-style boolean (0 = OK, <0 = ERROR) (-ERRNO) + */ +typedef int (config_handler_cb)(const char *input, mri_config_data_t *output); + +#endif diff --git a/src/mri_data.hpp b/src/mri_data.hpp index 8b97f51..a45a0c3 100644 --- a/src/mri_data.hpp +++ b/src/mri_data.hpp @@ -162,4 +162,7 @@ struct xpath_node { { mri_assert(parent || ! xdata); } }; +/* Root node is available for all */ +extern xpath_node mri_root; + #endif diff --git a/src/mri_general.hpp b/src/mri_general.hpp index 3df2de2..e7190d6 100644 --- a/src/mri_general.hpp +++ b/src/mri_general.hpp @@ -1,7 +1,9 @@ #ifndef _MRI_GENERAL_HPP_ #define _MRI_GENERAL_HPP_ -#include "mri_general.h" +extern "C" { + #include "mri_general.h" +} template struct c_return {}; diff --git a/src/mri_backend.cpp b/src/mri_producer.cpp similarity index 95% rename from src/mri_backend.cpp rename to src/mri_producer.cpp index fb7d37b..c58b95f 100644 --- a/src/mri_backend.cpp +++ b/src/mri_producer.cpp @@ -6,7 +6,7 @@ /* TODO(omer): Fix include paths */ extern "C" { - #include "mri.h" + #include "mri_producer.h" // #include "dn_common/general.h" } @@ -19,7 +19,9 @@ extern "C" { static mri_xtype_map_t mri_xtypes; static mri_shaper_map_t mri_shapers; static mri_formatter_map_t mri_formatters; -static xpath_node mri_root ("/", nullptr, nullptr, nullptr, nullptr); + +/* Root-Node externalized for all */ +xpath_node mri_root ("/", nullptr, nullptr, nullptr, nullptr); #define MRI_MAP_INSERT(map, key, ...) \ /* Note: ignore potential copy, even by using std::move, compiler optimization will fix this */ \ @@ -56,18 +58,6 @@ int xtype::add_slot(cstring_t slot, size_t offset, size_t size, mri_formatter_cb } extern "C" { - - int mri_init(cstring_t domain) { - MRI_SAFE_MAP_INSERTION( 0, - xpath_node, - mri_root.m_sub_nodes, - domain, - nullptr, - &mri_root, - nullptr, - nullptr); - } - /* --------------------------------------------------------------------------------------------- * Delegators to map user-defined format/shaper * --------------------------------------------------------------------------------------------- diff --git a/src/mri_service_discovery_factory.cpp b/src/mri_service_discovery_factory.cpp new file mode 100644 index 0000000..c73f3d7 --- /dev/null +++ b/src/mri_service_discovery_factory.cpp @@ -0,0 +1,36 @@ +#include + +#include "mri_general.hpp" + +enum class mock_enum { + REDIS_SERVICE_DISCOVERY, + ETCD_SERVICE_DISCOVERY, +}; + +/* Export code into mri_config.c properly */ + +extern "C" { + #include "mri_config_ext.h" + + /* Note: we hack the void* so CPP compiler is angry, fix this later */ + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wstrict-aliasing" + int config_to_service_discovery(const char *input, mri_config_data_t *output) { + std::string_view cpp_input(input); + if ("redis" == cpp_input) { + /* No need to allocate since enum fits generic pointer size */ + *((mock_enum *) &(output->data)) = mock_enum::REDIS_SERVICE_DISCOVERY; + output->length = sizeof(mock_enum); + } else if ("etcd" == cpp_input) { + /* No need to allocate since enum fits generic pointer size */ + *((mock_enum *) &(output->data)) = mock_enum::ETCD_SERVICE_DISCOVERY; + output->length = sizeof(mock_enum); + } else { + MRI_LOG(MRI_ERROR, "Did not find service discovery option for %s", input); + return -1; + } + + return 0; + } + #pragma GCC diagnostic pop +} From 1ea830c16fda0323d7e66fc1fb82cf573b54dc8e Mon Sep 17 00:00:00 2001 From: Omer Shamash Date: Sat, 9 May 2020 18:42:08 +0000 Subject: [PATCH 7/7] MRI: Great progress --- .ycm_extra_conf.py | 191 +++++++++++++++++++++++-------- UMakefile | 7 +- include/mri.h | 29 +++-- include/mri.hpp | 2 + include/mri_config.h | 12 ++ include/mri_producer.h | 15 +-- include/mri_typing.h | 2 + src/mri_config.c | 50 +++++++-- src/mri_data.hpp | 105 +++++++++++------ src/mri_producer.cpp | 204 +++++++++++++++++++++++++++------- src/mri_service_discovery.hpp | 11 +- src/mri_typing.hpp | 8 +- 12 files changed, 485 insertions(+), 151 deletions(-) diff --git a/.ycm_extra_conf.py b/.ycm_extra_conf.py index bebc38b..5a78855 100644 --- a/.ycm_extra_conf.py +++ b/.ycm_extra_conf.py @@ -33,13 +33,16 @@ import platform import os.path as p import subprocess +import json +import re +import logging + +logger = logging.getLogger('ycm-extra-conf') DIR_OF_THIS_SCRIPT = p.abspath( p.dirname( __file__ ) ) DIR_OF_THIRD_PARTY = p.join( DIR_OF_THIS_SCRIPT, 'third_party' ) SOURCE_EXTENSIONS = [ '.cpp', '.cxx', '.cc', '.c', '.m', '.mm' ] -database = None - # These are the compilation flags that will be used in case there's no # compilation database set (by default, one is not set). # CHANGE THIS LIST OF FLAGS. YES, THIS IS THE DROID YOU HAVE BEEN LOOKING FOR. @@ -57,12 +60,6 @@ '-DYCM_EXPORT=', # Custome flags '-Iinclude', -# THIS IS IMPORTANT! Without the '-x' flag, Clang won't know which language to -# use when compiling headers. So it will guess. Badly. So C++ headers will be -# compiled as C headers. You don't want that so ALWAYS specify the '-x' flag. -# For a C project, you would set this to 'c' instead of 'c++'. -'-x', -'c++', '-isystem', 'cpp/pybind11', '-isystem', @@ -91,12 +88,6 @@ 'cpp/ycm/benchmarks/benchmark/include', ] -# Clang automatically sets the '-std=' flag to 'c++14' for MSVC 2015 or later, -# which is required for compiling the standard library, and to 'c++11' for older -# versions. -if platform.system() != 'Windows': - flags.append( '-std=c++17' ) - # Set this to the absolute path to the folder (NOT the file!) containing the # compile_commands.json file to use that instead of 'flags'. See here for @@ -108,8 +99,6 @@ # # Most projects will NOT need to set this to anything; you can just change the # 'flags' list of compilation flags. Notice that YCM itself uses that approach. -compilation_database_folder = '' - def IsHeaderFile( filename ): extension = p.splitext( filename )[ 1 ] @@ -134,15 +123,58 @@ def PathToPythonUsedDuringBuild(): except OSError: return None +def dirwalk_up(bottom): + """ + mimic os.walk, but walk 'up' + instead of down the directory tree + """ + + bottom = p.realpath(bottom) + + #get files in current dir + try: + names = os.listdir(bottom) + except Exception as e: + print(e) + return + + + dirs = [] + files = [] + for name in names: + if p.isdir(p.join(bottom, name)): + dirs.append(name) + elif p.isfile(p.join(bottom, name)): + files.append(name) + + yield bottom, dirs, files + + new_path = p.realpath(p.join(bottom, '..')) + + # see if we are at the top + if new_path == bottom: + return + + for x in dirwalk_up(new_path): + yield x + +def FindNearBuildPath(filename): + file_dir = p.dirname(filename) + for path, dirs, files in dirwalk_up(file_dir): + if "compile_comands.json" in files: + return path + if path == os.getcwd(): + return None + for rdir in dirs: + if rdir == "build" and p.exists(rdir + "/compile_commands.json"): + return "{}/build".format(path) + return None + def Settings( **kwargs ): # Do NOT import ycm_core at module scope. import ycm_core - global database - if database is None and p.exists( compilation_database_folder ): - database = ycm_core.CompilationDatabase( compilation_database_folder ) - language = kwargs[ 'language' ] if language == 'cfamily': @@ -154,34 +186,105 @@ def Settings( **kwargs ): # in the corresponding source file. filename = FindCorrespondingSourceFile( kwargs[ 'filename' ] ) - if not database: - return { - 'flags': flags, - 'include_paths_relative_to_dir': DIR_OF_THIS_SCRIPT, - 'override_filename': filename - } + logger.info("Calculating flags for {}".format(filename)) - compilation_info = database.GetCompilationInfoForFile( filename ) - if not compilation_info.compiler_flags_: - return {} + # If we can't find compilation DB -> fallback to flags + compilation_database_folder = FindNearBuildPath(filename) + if not compilation_database_folder: + logger.info("Could not find build-path, using default falgs") + cpy_flags = flags + if any([filename.endswith(x) for x in ("c", "cc", "h", "hh")]): + cpy_flags += ['-x', 'c', '-std=c11'] + elif any([filename.endswith(x) for x in ("cpp", "cxx", "hpp", "hxx")]): + cpy_flags += ['-x', 'c++', '-std=c++17'] + return { + 'flags': cpy_flags, + 'include_paths_relative_to_dir': DIR_OF_THIS_SCRIPT, + 'override_filename': filename + } - # Bear in mind that compilation_info.compiler_flags_ does NOT return a - # python list, but a "list-like" StringVec object. - final_flags = list( compilation_info.compiler_flags_ ) - - # NOTE: This is just for YouCompleteMe; it's highly likely that your project - # does NOT need to remove the stdlib flag. DO NOT USE THIS IN YOUR - # ycm_extra_conf IF YOU'RE NOT 100% SURE YOU NEED IT. try: - final_flags.remove( '-stdlib=libc++' ) - except ValueError: - pass + logger.info("Found build-path, opening {}/compile_commands.json".format(compilation_database_folder)) + db_file = open(compilation_database_folder + "/compile_commands.json") + database = json.load(db_file) + entry = [e for e in database if e['file'].endswith(filename)] + if not entry: + logger.info("No entry for {} in compilation-db".format(filename)) + raise Exception("No entry") + logger.info("Found entry for {} in compilation-db".format(filename)) + entry = entry[0] + entry_fname = entry['file'] + entry_command = entry['command'] + entry_directory = entry['directory'] - return { - 'flags': final_flags, - 'include_paths_relative_to_dir': compilation_info.compiler_working_dir_, - 'override_filename': filename - } + iterable = iter(entry_command.split()) + final_flags = [] + RTE_INCLUDES + try: + current = next(iterable) + + # Skip until flags + clang_compilers = ("clang") + c_compilers = ("gcc", "c", "cc") + cpp_compilers = ("g++", "c++") + while True: + # already a flag - keep it + if current.startswith('-'): + break + # Skip filename + if current == entry_fname: + current = next(iterable) + break + # c-compiler, use -x flag to indicate clangd we are handling C file + if current in c_compilers or any([current.endswith("/" + c) for c in c_compilers]): + final_flags += ['-x', 'c'] + current = next(iterable) + break + # cpp-compiler, use -x flag to indicate clangd we are handling C++ file + elif current in cpp_compilers or any([current.endswith("/" + c) for c in cpp_compilers]): + final_flags += ['-x', 'c++'] + current = next(iterable) + break + # for clang, it should already contain the -x flag + elif current in clang_compilers or any([current.endswith("/" + c) for c in clang_compilers]): + current = next(iterable) + break + current = next(iterable) + + + while True: + # Defines, includes can be multi-param + if current in ("-D", "-I"): + final_flags += [current.append(next(iterable))] + # Double-skip these flags (-c file, -o file) + elif current in ("-o", "-c"): + current = next(iterable) + # Normal flag + else: + final_flags += [current] + current = next(iterable) + + + except StopIteration: + pass + + logger.info("Successfully loaded flags from compilation-db") + return { + 'flags': final_flags, + 'include_paths_relative_to_dir': relative_dir, + 'override_filename': filename + } + except: + logger.warn("Using default flags") + cpy_flags = flags + if any([filename.endswith(x) for x in ("c", "cc", "h", "hh")]): + cpy_flags += ['-x', 'c', '-std=c11'] + elif any([filename.endswith(x) for x in ("cpp", "cxx", "hpp", "hxx")]): + cpy_flags += ['-x', 'c++', '-std=c++17'] + return { + 'flags': cpy_flags, + 'include_paths_relative_to_dir': DIR_OF_THIS_SCRIPT, + 'override_filename': filename + } if language == 'python': return { diff --git a/UMakefile b/UMakefile index 3f53483..c1ce64b 100644 --- a/UMakefile +++ b/UMakefile @@ -14,7 +14,8 @@ $lflags = -Wl,-zdefs -fPIC $mri_includes = -Iinclude -Ideps $mri_cxx_flags = -fno-exceptions +$mri_linkage = -lhiredis -:foreach src/*.c > !c($mri_includes) -:foreach src/*.cpp > !cpp($mri_includes, $mri_cxx_flags) -: src/*.umake.o > !so( ) > libmri.so +:foreach **/*.c > !c($mri_includes) +:foreach **/*.cpp > !cpp($mri_includes, $mri_cxx_flags) +: **/*.umake.o > !so($mri_linkage) > libmri.so diff --git a/include/mri.h b/include/mri.h index 98165f8..1ff10b2 100644 --- a/include/mri.h +++ b/include/mri.h @@ -1,21 +1,36 @@ #ifndef _MRI_H_ #define _MRI_H_ +#include "mri_sched.h" #include "mri_config.h" #include "mri_typing.h" #include "mri_logging.h" /* prototypes */ -/** - * mri_init() - * - * Notes: +/* Notes: * Each instance can only be linked to one domain (! consider !) * Instance is global to process having thread-oriented CB support + */ + +/** + * mri_init_threaded() + * Initialize the MRI instance in threaded-mode (new thread is created) + * All of the events needed to be handled by MRI will be done in the thread + * + * @param config - pointer to configuration object to use (ownership transfered) + * @return int - C-Style boolean (0 = OK, <0 = ERROR) (-ERRNO) + */ +int mri_init_threaded(struct mri_coniguration **config); + +/** + * mri_init_async() + * Initialize the MRI instance in async-mode (no threads are created) + * All of the events needed to be handled by MRI will happen in user-thread * - * @param config - pointer to configuration object to use (ownership transfered) - * @returns C-Style boolean (0 = OK, <0 = ERROR) (-ERRNO) + * @param config - pointer to configuration object to use (ownership transfered) + * @param sched - information needed to schedule MRI events into user-scheduling + * @return int - C-Style boolean (0 = OK, <0 = ERROR) (-ERRNO) */ -int mri_init(struct mri_coniguration **config); +int mri_init_async(struct mri_coniguration **config, mri_sched_info_t *sched); #endif /* _MRI_H_ */ diff --git a/include/mri.hpp b/include/mri.hpp index c354888..7b77353 100644 --- a/include/mri.hpp +++ b/include/mri.hpp @@ -17,6 +17,8 @@ template < typename container_t, typename iterator_t = typename container_t::iterator, typename value_t = typename container_t::value_type > int generic_mri_iterator_cb(void *container, mri_iter_state_t *state, void **mem) { + if (! container || ! state || ! mem) return MRI_ITER_ERR; + /* We are getting the templated type as data */ auto const &data_view = *((container_t *) container); diff --git a/include/mri_config.h b/include/mri_config.h index c331bb5..154d132 100644 --- a/include/mri_config.h +++ b/include/mri_config.h @@ -17,10 +17,22 @@ typedef enum mri_config_value { MRI_CONFIG_SERVICE_DISCOVERY_PORT, MRI_CONFIG_SERVICE_DISCOVERY_TO_USE, + /* Redis specific */ + MRI_CONFIG_REDIS_SD_TIMEOUT, + MRI_CONFIG_REDIS_SD_USE_SENTINEL, + MRI_CONFIG_REDIS_SD_SENTINEL_MASTER, + /* Never go beyond this point */ MRI_CONFIG_MAX_VALUE } mri_config_value_t; +/** + * Convert key to human-readable string (log usage) + * @param type - the enumeration value for the key + * @return cstring_t - string representing the enum-value + */ +cstring_t config_type_to_string(mri_config_value_t type); + /** * Create configuration object * Input is formatted in getopt() fashion diff --git a/include/mri_producer.h b/include/mri_producer.h index ddddf13..a0f95c8 100644 --- a/include/mri_producer.h +++ b/include/mri_producer.h @@ -1,6 +1,7 @@ #ifndef _MRI_PRODUCER_H_ #define _MRI_PRODUCER_H_ +#include "mri_sched.h" #include "mri_typing.h" /* Formatting apart from provided-formatters section */ @@ -18,15 +19,15 @@ int _mri_type_add_vslot(cstring_t, cstring_t, mri_formatter_cb); int _mri_type_add_shaper(cstring_t, cstring_t, cstring_t, size_t, size_t); /* Current suggestion: - * when someone registers data on path, we capture his thread-id and - * allow him to poll on an event object that is related to his thread-id + * when someone registers data on path, we capture his thread-id, + * this will allow for poll on an event object that is related to his thread-id. + * MRI will use data in to register the event-object for the user to poll on. + * Ultimatly this will allow user to synchronise access to registered data without (b)locking. * - * mri_get_eventloop_fd() will allow user to allocate time for MRI in any bitch-thread - * mri_get_notification_fd() will allow to get the thread-bound event-fd to poll in relevant thread + * If none are provided, thread's data will use the default MRI sched (provided in init, or mri-thread) */ -int mri_get_eventloop_fd(); -int mri_get_notification_fd(); - +int mri_set_current_thread_sched(mri_sched_info_t *sched); + /* Registering data of defined types to MRI */ int _mri_register(cstring_t path, cstring_t type, void *object, mri_iterator_cb callback); int mri_register_config(cstring_t path, void *context, mri_config_change_cb callback); diff --git a/include/mri_typing.h b/include/mri_typing.h index 494c8a8..bb726ed 100644 --- a/include/mri_typing.h +++ b/include/mri_typing.h @@ -3,6 +3,7 @@ #include #include +#include /* defines */ #define MRI_MAX_HOST_SIZE (128) @@ -20,6 +21,7 @@ typedef enum mri_service_mode { } mri_service_mode_t; typedef enum mri_iter_action { + MRI_ITER_ERR, MRI_ITER_STOP, MRI_ITER_CONTINUE } mri_iter_action_t; diff --git a/src/mri_config.c b/src/mri_config.c index b0fd93b..83e5848 100644 --- a/src/mri_config.c +++ b/src/mri_config.c @@ -8,12 +8,15 @@ #include "mri_config_ext.h" /* Note: remember to update to have proper debugging later */ -const char *config_type_to_string(mri_config_value_t type) { +cstring_t config_type_to_string(mri_config_value_t type) { switch(type) { case MRI_CONFIG_DOMAIN : return "DOMAIN"; case MRI_CONFIG_SERVICE_DISCOVERY_HOST : return "SERVICE_DISCOVERY_HOST"; case MRI_CONFIG_SERVICE_DISCOVERY_PORT : return "SERVICE_DISCOVERY_PORT"; case MRI_CONFIG_SERVICE_DISCOVERY_TO_USE: return "SERVICE_DISCOVERY_TO_USE"; + case MRI_CONFIG_REDIS_SD_TIMEOUT : return "REDIS_SD_TIMEOUT"; + case MRI_CONFIG_REDIS_SD_USE_SENTINEL : return "REDIS_SD_USE_SETINEL"; + case MRI_CONFIG_REDIS_SD_SENTINEL_MASTER: return "REDIS_SD_SETINEL_MASTER"; default : return "UNKNOWN"; } } @@ -22,32 +25,55 @@ struct mri_configuration { mri_config_data_t config[MRI_CONFIG_MAX_VALUE]; }; -int cstring_ref_handler(const char *input, mri_config_data_t *output) { +int config_to_cstring_ref(const char *input, mri_config_data_t *output) { output->data = (void *) input; output->length = strnlen(input, MRI_CONFIG_MAX_STRING_SIZE); return 0; } +#include +int config_to_boolean(const char *input, mri_config_data_t *output) { + if (0 == strcasecmp(input, "true")) { + output->data = (void *) 1; + } else if (0 == strcasecmp(input, "false")) { + output->data = (void *) 0; + } else { + MRI_LOG(MRI_ERROR, "Cannot parse boolean config (%s), not false/true", input); + return -1; + } + + output->length = sizeof(int); + return 0; +} + /* External handlers from project */ -extern int config_to_service_discovery(const char *, mri_config_data_t *); +extern int config_to_timeval (const char *, mri_config_data_t *); +extern int config_to_service_discovery (const char *, mri_config_data_t *); /* Array of typedef int (config_handler_cb)(const char *input, mri_config_data_t *output); */ static int (*handle_config_cb[MRI_CONFIG_MAX_VALUE])(const char *, mri_config_data_t *) = { /* Note: we assume input is from main() so no need to allocate */ - [MRI_CONFIG_DOMAIN] = cstring_ref_handler, - [MRI_CONFIG_SERVICE_DISCOVERY_HOST] = cstring_ref_handler, - [MRI_CONFIG_SERVICE_DISCOVERY_PORT] = cstring_ref_handler, - [MRI_CONFIG_SERVICE_DISCOVERY_TO_USE] = config_to_service_discovery + [MRI_CONFIG_DOMAIN] = config_to_cstring_ref, + [MRI_CONFIG_SERVICE_DISCOVERY_HOST] = config_to_cstring_ref, + [MRI_CONFIG_SERVICE_DISCOVERY_PORT] = config_to_cstring_ref, + [MRI_CONFIG_SERVICE_DISCOVERY_TO_USE] = config_to_service_discovery, + /* Redis related */ + [MRI_CONFIG_REDIS_SD_TIMEOUT] = config_to_timeval, + [MRI_CONFIG_REDIS_SD_USE_SENTINEL] = config_to_boolean, + [MRI_CONFIG_REDIS_SD_SENTINEL_MASTER] = config_to_cstring_ref }; static int parse_opts(struct mri_configuration *conf, int argc, char **argv) { int option_index; struct option long_options[] = { - { "domain", required_argument, 0, MRI_CONFIG_DOMAIN }, - { "sd-host", required_argument, 0, MRI_CONFIG_SERVICE_DISCOVERY_HOST }, - { "sd-port", required_argument, 0, MRI_CONFIG_SERVICE_DISCOVERY_PORT }, - { "service-discovery", required_argument, 0, MRI_CONFIG_SERVICE_DISCOVERY_TO_USE }, - { 0, 0, 0, 0 } + { "domain", required_argument, 0, MRI_CONFIG_DOMAIN }, + { "sd-host", required_argument, 0, MRI_CONFIG_SERVICE_DISCOVERY_HOST }, + { "sd-port", required_argument, 0, MRI_CONFIG_SERVICE_DISCOVERY_PORT }, + { "service-discovery", required_argument, 0, MRI_CONFIG_SERVICE_DISCOVERY_TO_USE }, + { "redis-timeout", required_argument, 0, MRI_CONFIG_REDIS_SD_TIMEOUT }, + { "redis-with-sentinel", required_argument, 0, MRI_CONFIG_REDIS_SD_USE_SENTINEL }, + { "redis-sentinel-master", required_argument, 0, MRI_CONFIG_REDIS_SD_SENTINEL_MASTER }, + { 0, 0, 0, 0 } }; /* Fetch first option and loop */ diff --git a/src/mri_data.hpp b/src/mri_data.hpp index a45a0c3..8e26f02 100644 --- a/src/mri_data.hpp +++ b/src/mri_data.hpp @@ -2,16 +2,14 @@ #define _MRI_DATA_HPP_ #include +#include #include #include +#include "mri_typing.hpp" -/* gettid() is undefined in glibc, use this instead */ -#include -#include +/* glibc doesn't declare tid_t properly */ +#include typedef pid_t tid_t; -static tid_t _gettid() { return syscall(SYS_gettid); } - -#include "mri_typing.hpp" /* TODO(omer): Fix this */ #define mri_assert(cond) (void) 0 @@ -22,14 +20,19 @@ struct xformat; struct xshaper; struct mri_slot; struct xpath_node; +struct mri_thread_sched; struct slot_capture_data; -using mri_slot_map_t = mri_ordered_map; -using mri_subpath_map_t = mri_ordered_map; -using mri_capture_map_t = mri_unordered_map; -using mri_formatter_map_t = mri_unordered_map; -using mri_shaper_map_t = mri_unordered_map; -using mri_xtype_map_t = mri_unordered_map; +using mri_slot_map_t = mri_ordered_map>; +using mri_subpath_map_t = mri_ordered_map>; +using mri_capture_map_t = mri_unordered_map>; +using mri_formatter_map_t = mri_unordered_map>; +using mri_shaper_map_t = mri_unordered_map>; +using mri_xtype_map_t = mri_unordered_map>; +using mri_thread_sched_map_t = mri_ordered_map>; +using mri_result_t = mri_array; +using mri_result_set_t = mri_vector; +using mri_node_result_set_t = mri_vector; struct xformat { std::string m_name; @@ -119,6 +122,28 @@ struct xtype { * @return C-Style boolean (0 = OK, <0 = ERROR) (-ERRNO) */ int add_slot(cstring_t slot, size_t offset, size_t size, mri_formatter_cb formatter, int flags); + + /** + * Dump the data based on this type + * @param input - input data that coresponds to this type + * @param rs - result set to add into (one result per-slot) + * @return bool - did function work properly + */ + bool xdump(void *input, mri_result_set_t &rs); +}; + +struct xnode_get { + tid_t m_tid; /*>! Thread whom registered node (origin) */ + xtype *m_type; /*>! Type coresponding with the node's data */ + void *m_xdata; /*>! User-defined data (node data internal) */ + mri_iterator_cb m_xdata_iterator; /*>! User-defined data iterator (tableview) */ +}; + +struct xnode_set { + tid_t m_tid; /*>! Thread whom registered node (origin) */ + xtype *m_type; /*>! Type coresponding with the node's data */ + void *m_xcontext; /*>! User-defined data (xcallbck's context) */ + mri_config_change_cb m_xcallback; /*>! User-defined callback to handle data */ }; /** @@ -127,39 +152,55 @@ struct xtype { */ struct xpath_node { /* General definition of a node */ - std::string m_name; /*>! Name of node (name unique per parent) */ - mri_subpath_map_t m_sub_nodes; /*>! Mapping of sub-nodes (each is unique) */ - xtype *m_type; /*>! Type coresponding with the node's data */ - xpath_node *m_parent; /*>! Reverse reference to the parent's node */ + std::string m_name; /*>! Name of node (name unique per parent) */ + mri_subpath_map_t m_sub_nodes; /*>! Mapping of sub-nodes (each is unique) */ + xpath_node *m_parent; /*>! Reverse reference to the parent's node */ /* User-data (specified on registration) */ - tid_t m_tid; /*>! Thread whom registered node (origin) */ - void *m_xdata; /*>! User-defined data (node data internal) */ - mri_iterator_cb m_xdata_iterator; /*>! User-defined data iterator (tableview) */ + xnode_get m_get_data; /*>! Get context (used for node-querying) */ + xnode_set m_set_data; /*>! Set context (used for node-configuring) */ /** * Notes: * 1. No type = empty node, path-building * 2. No parent = root node, others should not do this - * 3. No data = only allowed for empty node - * 4. Iterator = optional, data can be flat - * 5. Name cannot contain / (except root) to allow path traversal - * 6. Path is defined as ///.../ + * 3. get_data/set_data - filled later by registering function + * 4. Name cannot contain / (except root) to allow path traversal + * 5. Path is defined as ///.../ */ template xpath_node( string_t &&name, - xtype *type, - xpath_node *parent, - void *xdata, - mri_iterator_cb data_iter) + xpath_node *parent) : m_name(std::forward(name)) , m_sub_nodes() - , m_type(type) , m_parent(parent) - , m_tid(_gettid()) - , m_xdata(xdata) - , m_xdata_iterator(data_iter) - { mri_assert(parent || ! xdata); } + , m_get_data() + , m_set_data() + {} + + /** + * get_fully_qualified_path() + * Most likely used for logging and debugging + * @return std::string - full path of the node + */ + std::string get_fully_qualified_path() { + if (! m_parent) return ""; + return m_parent->get_fully_qualified_path() + "/" + m_name; + } + + /** + * Fetch requested sub-node (get-create mode) + * @param name - string_view representing name of the node requested (avoid malloc) + * @return xpath_node const * - pointer to the newly created sub-node (nullptr on failure) + */ + xpath_node const *fetch_subnode(std::string_view name); + + /** + * Dump the node-data into the result-set + * @param rs - reference to result-set to output into + * @return bool - did we add anything into result-set + */ + bool xdump(mri_node_result_set_t &rs); }; /* Root node is available for all */ diff --git a/src/mri_producer.cpp b/src/mri_producer.cpp index c58b95f..d14819b 100644 --- a/src/mri_producer.cpp +++ b/src/mri_producer.cpp @@ -1,47 +1,58 @@ #include +#include #include #include +#include #include #include -/* TODO(omer): Fix include paths */ +#include +#include +#include + extern "C" { #include "mri_producer.h" // #include "dn_common/general.h" } +#include "mri_data.hpp" +#include "mri_sched.hpp" #include "mri_general.hpp" //#include "../deps/ordered_map.h" -#include "mri_data.hpp" +/* gettid() is undefined in glibc, use this instead */ +#include +static tid_t _gettid() { return syscall(SYS_gettid); } /* Global variables */ -static mri_xtype_map_t mri_xtypes; -static mri_shaper_map_t mri_shapers; -static mri_formatter_map_t mri_formatters; +static mri_xtype_map_t mri_xtypes; +static mri_shaper_map_t mri_shapers; +static mri_formatter_map_t mri_formatters; /* Root-Node externalized for all */ -xpath_node mri_root ("/", nullptr, nullptr, nullptr, nullptr); - -#define MRI_MAP_INSERT(map, key, ...) \ - /* Note: ignore potential copy, even by using std::move, compiler optimization will fix this */ \ - /* Note: usage of m_name.c_str() as the map-key allows us to avoid another string allocation */ \ - decltype(map)::mapped_type __temp_cls (key, ##__VA_ARGS__); \ - auto&& [iter, did_insert] = map.try_emplace(__temp_cls.m_name.c_str(), std::move(__temp_cls)) - -#define MRI_MAP_INSERT_CHECK_RES(exists_err, map_data_t, map, key) \ - if (likely(did_insert)) return 0; /* Bailout if all worked */ \ - \ - if (map.end() == iter) { \ - MRI_LOG(MRI_ERROR, "Failed to insert " #key " %s " #map_data_t " (OOM)", key); \ - return -ENOMEM; \ - } \ - \ - MRI_LOG(MRI_WARNING, #map_data_t " %s already exists", key); \ - return exists_err - -#define MRI_SAFE_MAP_INSERTION(exists_err, map_data_t, map, key, ...) \ - MRI_MAP_INSERT(map, key, ##__VA_ARGS__); \ +xpath_node mri_root ("/", nullptr); + +#define MRI_MAP_INSERT(map, key, ...) \ + /* Note: ignore potential copy, even by using std::move, compiler optimization will fix this */ \ + /* Note: usage of m_name.c_str() as the map-key allows us to avoid another string allocation */ \ + auto __temp_cls = std::make_unique(key, ##__VA_ARGS__); \ + auto&& [iter, did_insert] = map.try_emplace(__temp_cls->m_name.c_str(), std::move(__temp_cls)) + +#define MRI_MAP_INSERT_CHECK_RES(exists_err, map_data_t, map, key) \ + do { \ + if (unlikely(! did_insert)) { \ + if (map.end() == iter) { \ + MRI_LOG(MRI_ERROR, "Failed to insert " #key " %s " #map_data_t " (OOM) (ˆ⺫ˆ๑)<3",key); \ + return -ENOMEM; \ + } \ + \ + MRI_LOG(MRI_WARNING, #map_data_t " %s already exists", key); \ + return exists_err; \ + } \ + } while (0) + +#define MRI_SAFE_MAP_INSERTION(exists_err, map_data_t, map, key, ...) \ + MRI_MAP_INSERT(map, key, ##__VA_ARGS__); \ MRI_MAP_INSERT_CHECK_RES(exists_err, map_data_t, map, key) /* Easier to implement here due to macros, consider moving */ @@ -55,6 +66,7 @@ int xtype::add_slot(cstring_t slot, size_t offset, size_t size, mri_formatter_cb size, formatter, (flags & MRI_SLOT_FLAG_HIDDEN)); + return 0; } extern "C" { @@ -64,19 +76,27 @@ extern "C" { */ int _mri_create_formatter(cstring_t formatter, mri_formatter_cb callback) { + /* Sanity check */ + if (! formatter || ! callback) return -EINVAL; + MRI_SAFE_MAP_INSERTION( -EEXIST, xformat, mri_formatters, formatter, callback); + return 0; } int _mri_create_shaper(cstring_t shaper, mri_shaper_cb callback) { + /* Sanity check */ + if (! shaper || ! callback) return -EINVAL; + MRI_SAFE_MAP_INSERTION( -EEXIST, xshaper, mri_shapers, shaper, callback); + return 0; } /* --------------------------------------------------------------------------------------------- @@ -85,11 +105,15 @@ extern "C" { */ int _mri_create_type(cstring_t type, size_t size) { + /* Sanity check */ + if (! type) return -EINVAL; + MRI_SAFE_MAP_INSERTION( 0, xtype, mri_xtypes, type, size); + return 0; } @@ -99,21 +123,24 @@ extern "C" { size_t offset, size_t size, int slot_flags) { + /* Sanity check */ + if (! type || ! name || ! formatter) return -EINVAL; + auto xtype_iter = mri_xtypes.find(type); if (unlikely(mri_xtypes.end() == xtype_iter)) { - MRI_LOG(MRI_ERROR, "xtype %s does not exist, better luck next time", type); + MRI_LOG(MRI_ERROR, "xtype %s does not exist, better luck next time (҂◡_◡) ᕤ", type); return -EINVAL; } auto xformat_iter = mri_formatters.find(formatter); if (unlikely(mri_formatters.end() == xformat_iter)) { - MRI_LOG(MRI_ERROR, "xformat %s does not exist, better luck next time", formatter); + MRI_LOG(MRI_ERROR, "xformat %s does not exist, better luck next time (҂◡_◡) ᕤ", formatter); return -EINVAL; } auto &mri_type = xtype_iter->second; auto const &format = xformat_iter->second; - return mri_type.add_slot(name, offset, size, format.m_formatter, slot_flags); + return mri_type->add_slot(name, offset, size, format->m_formatter, slot_flags); } /** @@ -125,6 +152,9 @@ extern "C" { * @param callback - user-defined function to format vslot */ int _mri_type_add_vslot(cstring_t type, cstring_t name, mri_formatter_cb callback) { + /* Sanity check */ + if (! type || ! name || ! callback) return -EINVAL; + auto xtype_iter = mri_xtypes.find(type); if (unlikely(mri_xtypes.end() == xtype_iter)) { MRI_LOG(MRI_ERROR, "xtype %s does not exist, better luck next time", type); @@ -149,7 +179,7 @@ extern "C" { auto const &format = iter->second; auto &mri_type = xtype_iter->second; - return mri_type.add_slot(name, 0, mri_type.m_size, format.m_formatter, 0); + return mri_type->add_slot(name, 0, mri_type->m_size, format->m_formatter, 0); } int _mri_type_add_shaper( cstring_t type, @@ -157,46 +187,140 @@ extern "C" { cstring_t shaper, size_t offset, size_t size) { + /* Sanity check */ + if (! type || ! slot || ! shaper) return -EINVAL; + auto xtype_iter = mri_xtypes.find(type); if (unlikely(mri_xtypes.end() == xtype_iter)) { - MRI_LOG(MRI_ERROR, "xtype %s does not exist, better luck next time", type); + MRI_LOG(MRI_ERROR, "xtype %s does not exist, better luck next time (҂◡_◡) ᕤ", type); return -EINVAL; } auto shaper_iter = mri_shapers.find(shaper); if (mri_shapers.end() == shaper_iter) { - MRI_LOG(MRI_ERROR, "shaper %s does not exist, better luck next time", shaper); + MRI_LOG(MRI_ERROR, "shaper %s does not exist, better luck next time (҂◡_◡) ᕤ", shaper); return -EINVAL; } auto &mri_type = xtype_iter->second; /* Add hidden slot for shaping (allow for user to add it as not-hidden before) */ - int retval = mri_type.add_slot(slot, offset, size, nullptr, MRI_SLOT_FLAG_HIDDEN); + int retval = mri_type->add_slot(slot, offset, size, nullptr, MRI_SLOT_FLAG_HIDDEN); if (unlikely(0 != retval && -EEXIST != retval)) return retval; /* Save shaper to use later in xdump */ auto const &xshaper = shaper_iter->second; - auto &xslot = mri_type.m_slots.find(slot)->second; - xslot.m_shaper = xshaper.m_shaper; - auto&& [iter, did_insert] = mri_type.m_slot_capture_map.try_emplace(slot, slot_capture_data {}); - MRI_MAP_INSERT_CHECK_RES(-EEXIST, slot_capture_data, mri_type.m_slot_capture_map, slot); + auto &xslot = mri_type->m_slots.find(slot)->second; + xslot->m_shaper = xshaper->m_shaper; + auto __temp_cls = std::make_unique(); + auto&& [iter, did_insert] = mri_type->m_slot_capture_map.try_emplace(slot, std::move(__temp_cls)); + MRI_MAP_INSERT_CHECK_RES(-EEXIST, slot_capture_data, mri_type->m_slot_capture_map, slot); + return 0; } /* --------------------------------------------------------------------------------------------- * Event-loop and timing related functions * --------------------------------------------------------------------------------------------- */ + int mri_set_current_thread_sched(mri_sched_info_t *sched) { + /* Sanity check */ + if (! sched || ! sched->register_event || ! sched->unregister_event) return -EINVAL; - int mri_get_eventloop_fd(); - int mri_get_notification_fd(); + /* Register sub-scheduler for current thread's ID */ + return mri_main_sched::instance().register_subsched(_gettid(), *sched) ? 0 : -1; + } /* --------------------------------------------------------------------------------------------- * Custom user-data registration related functions * --------------------------------------------------------------------------------------------- */ - int _mri_register(cstring_t path, cstring_t type, void *object, mri_iterator_cb callback); + xpath_node *mri_create_path_from_root(cstring_t path) { + /* Sanity */ + if ('/' != *path) { + MRI_LOG(MRI_ERROR, "Path (%s) does not start at root ୧༼ಠ益ಠ༽୨", path); + return nullptr; + } + + xpath_node *node = &mri_root; + cstring_t iter = path + 1, last_node = path; + for (; *iter; ++iter) { + /* Note: + * Node starts after last '/' ( + 1) + * and ends before following '/' ( - 1) + */ + if ('/' == *iter) { + /* Calculate length of node */ + intptr_t str_len = (intptr_t) iter - (intptr_t) last_node - 1; + + /* Skip empty nodes (non-strict format) */ + if (0 >= str_len) { + ++last_node; + continue; + } + + /* Actual name of the node */ + std::string_view node_name { last_node + 1, (size_t) str_len }; + + /* Update current node indicator to new subnode */ + node = (xpath_node *) node->fetch_subnode(node_name); + if (! node) { + /* Error already logged by fetch_subnode() */ + /* TODO(omer): Support rollback ? */ + return nullptr; + } + + /* Update last */ + last_node = iter; + } + } + + /* Fetch the leaf - actual work needed */ + + /* Calculate length of node */ + intptr_t str_len = (intptr_t) iter - (intptr_t) last_node - 1; + + /* Last node is empty -> return last known (non-strict) */ + if (0 >= str_len) return node; + + std::string_view node_name { last_node + 1, (size_t) str_len }; + node = (xpath_node *) node->fetch_subnode(node_name); + if (! node) { + /* Error already logged by fetch_subnode() */ + /* TODO(omer): Support rollback ? */ + return nullptr; + } + + return node; + } + + int _mri_register(cstring_t path, cstring_t type, void *object, mri_iterator_cb callback) { + if (! path || ! type || ! callback) return -EINVAL; + + /* Find relevant type */ + auto xtype_iter = mri_xtypes.find(type); + if (unlikely(mri_xtypes.end() == xtype_iter)) { + MRI_LOG(MRI_ERROR, "xtype %s does not exist, better luck next time (҂◡_◡) ᕤ", type); + return -EINVAL; + } + + /* Get underlying node (creating path in the process) */ + xpath_node *leaf_node = mri_create_path_from_root(path); + if (! leaf_node) { + /* Error already logged by internal call to fetch_subnode() */ + return -ENOMEM; + } + + /* Fill in leaf data (get-format) */ + leaf_node->m_get_data = xnode_get { + .m_tid = _gettid(), + .m_type = xtype_iter->second.get(), + .m_xdata = object, + .m_xdata_iterator = callback + }; + + return 0; + } int mri_register_config(cstring_t path, void *context, mri_config_change_cb callback); int mri_unregister(cstring_t path); } diff --git a/src/mri_service_discovery.hpp b/src/mri_service_discovery.hpp index 8c0ad09..eda092c 100644 --- a/src/mri_service_discovery.hpp +++ b/src/mri_service_discovery.hpp @@ -1,9 +1,8 @@ #ifndef _MRI_SERVICE_DISCOVERY_HPP_ #define _MRI_SERVICE_DISCOVERY_HPP_ -extern "C" { - #include "mri_config.h" -} +#include +#include "mri_config.hpp" #include "mri_typing.hpp" /* This will change everytime we change the mri_service_data structure */ @@ -11,24 +10,26 @@ extern "C" { struct mri_service_data { uint32_t version; /*!< service data versioning first ! */ char host[MRI_MAX_HOST_SIZE]; /*!< host containing the MRI instance */ + uint16_t port; /*!< port used to expose the MRI data */ mri_service_mode_t mode; /*!< MRI usage mode (producer/consumer) */ }; /* Data describing current service */ extern mri_service_data self; -using mri_service_map = mri_ordered_map; +using mri_service_map = mri_ordered_map>; class IServiceDiscovery { protected: IServiceDiscovery() = default; + virtual ~IServiceDiscovery() = default; public: IServiceDiscovery(IServiceDiscovery&) = delete; IServiceDiscovery(IServiceDiscovery&&) = delete; /* Responsible add `self` if it decides that it is needed */ virtual bool initialize(mri_configuration &mri_config) = 0; - virtual mri_service_map &fetch_services() = 0; + virtual mri_service_map const &fetch_services() const = 0; }; #endif diff --git a/src/mri_typing.hpp b/src/mri_typing.hpp index 65d1a32..350edd7 100644 --- a/src/mri_typing.hpp +++ b/src/mri_typing.hpp @@ -1,9 +1,12 @@ #ifndef _MRI_TYPING_HPP_ #define _MRI_TYPING_HPP_ -#include "mri_typing.h" +extern "C" { + #include "mri_typing.h" +} #include +#include #include #include @@ -16,4 +19,7 @@ using mri_ordered_map = std::map; template using mri_vector = std::vector; +template +using mri_array = std::array; + #endif