docs/docs/architecture-overview.md

🏗️ Architecture Overview

This document provides an overview of the system architecture and its major components.

---

Table of Contents

• System Context
• Component Diagrams
  • Components in Elixier Management Tool
  • Components in Kubernetes Cluster

---

System Context

C4Context

System(docs, "Documentation", "Gitea Pages")
Person(admins, "Admins", "Tool administrators<br>e.g Abyres")
System(emgr, "Elixier Management Tool", "Web app")

Enterprise_Boundary(eb1, "Client") {
    Person_Ext(clients, "Clients", "Tool & data platform users<br>e.g Credence")
    System_Boundary(sb1, "On-prem / cloud") {
        System_Ext(sso, "Identity Provider", "Single-sign on<br>user auth")
        System_Ext(k3s, "Kubernetes Cluster", "Infra & compute power<br>for data platform")
        System_Ext(s3, "Object Storage", "Store raw &<br>processed data")
    }
}

Rel(admins, emgr, "Manage<br>clients")
Rel(admins, docs, "Refer")

Rel(clients, docs, "Refer")
Rel(clients, emgr, "Manage tenants, hosts,<br>clusters & services")
Rel(clients, k3s, "Use services", "e.g Superset & Airflow")

Rel(emgr, k3s, "Deploy clusters<br>& services")
Rel(emgr, s3, "Read/write<br>datasets & ETL")
BiRel(emgr, sso, "SSO", "e.g OIDC")

Rel(k3s, s3, "Read &<br>write files")
BiRel(k3s, sso, "SSO", "e.g OIDC")

---

Component Diagrams

Components in Elixier Management Tool

C4Component

Person(admins, "Admins", "Tool administrators<br>e.g Abyres")
Person_Ext(clients, "Clients", "Tool & data platform users<br>e.g Credence")

Container_Boundary(emgr, "Elixier Management Tool") {
    Component(frontend, "Frontend", "Reflex, NextJS", "Web app frontend")
    Component(backend, "Backend", "Reflex, FastAPI", "Web app backend")
    ComponentDb(postgres, "Database", "PostgreSQL", "Store app data &<br>cluster metadata")
    ComponentDb(opensearch, "Log Store", "OpenSearch", "Store logs of hosts,<br>clusters, services & audit")
    ComponentQueue(valkey, "Message Queue", "Valkey", "Send commands to agent")
}
Component_Ext(sso, "Identity Provider", "e.g Authentik / Keycloak")
Component_Ext(s3, "Object Storage", "e.g MinIO / S3")

Rel(admins, frontend, "Use", "HTTPS")
Rel(clients, frontend, "Use", "HTTPS")

BiRel(frontend, backend, "Exchange data", "Websocket, HTTPS")
Rel(backend, sso, "SSO", "OIDC")
BiRel(backend, postgres, "Read/write data")
BiRel(backend, opensearch, "Read/write data")
Rel(backend, s3, "Read/write<br>datasets & ETL", "S3 API")
Rel(backend, valkey, "Queue commands")

Components in Kubernetes Cluster

C4Component

Person_Ext(clients, "Clients", "Tool & data platform users<br>e.g Credence")

Container_Boundary(k3s, "Kubernetes Cluster") {
    Container_Ext(superset, "Data Visualization", "Superset", "Build charts & dashboards")
    Container_Ext(airflow, "Data ETL", "Airflow", "Run ETL scripts")
    Container_Ext(trino, "SQL Engine", "Trino", "Read & write structured data<br>to object storage")
    Container(agent, "Agent", "EMGR Operator", "Execute commands,<br>send metrics data")
}
Container_Boundary(emgr, "Elixier Management Tool") {
    Container(backend, "Backend", "Reflex, FastAPI", "Web app backend")
}
Container_Ext(sso, "Identity Provider", "e.g Authentik / Keycloak")
Container_Ext(s3, "Object Storage", "e.g MinIO / S3")

Rel(clients, superset, "Use", "HTTPS")
Rel(clients, airflow, "Use", "HTTPS")

Rel(trino, s3, "Read/write data", "Hive, Iceberg")
Rel(superset, trino, "Retrieve data", "SQL")
Rel(airflow, trino, "Read/write data", "SQL")
Rel(airflow, s3, "Read/write data", "S3 API")
Rel(superset, sso, "SSO", "OIDC")
Rel(airflow, sso, "SSO", "OIDC")
BiRel(agent, backend, "Send commands,<br>metrics data")