workspace "Realtime smart cities" "C4 Container model: Data -> Platform -> ThingsBoard; Predictive -> TDengine, Analytical -> Neo4j" {

  model {
    operator = person "Operator" "Views dashboards and reacts to alerts."

    social = softwareSystem "Social Networks" "External social feeds, APIs, webhooks." {
      tags "External, DataSource"
    }
    iot = softwareSystem "IoT Data" "Devices/gateways sending telemetry." {
      tags "External, DataSource"
    }
    sys = softwareSystem "Sensors & Infrastructure (Syslog)" "Network devices, servers, and sensors emitting RFC3164/RFC5424 syslogs." {
      tags "External, DataSource"
    }

    rtsa = softwareSystem "Streaming Platform" "Processes real-time data and serves dashboards." {

      group "Ingress & Messaging" {
        mqtt = container "MQTT Broker" "Broker for IoT device telemetry & commands." "EMQX/HiveMQ/Mosquitto" {
          tags "MessageBroker"
        }
        kconnect = container "Kafka Connect" "Runs connectors to move data in/out of Kafka." "Kafka Connect" {
          tags "Integration"
          mqtt_source = component "MQTT Source Connector" "Subscribes to MQTT topics; writes to Kafka 'iot.telemetry'." "Connector"
        }

        kafka_ingest = container "Apache Kafka (Ingress)" "Raw topics from external producers." "Apache Kafka" {
          tags "EventStreaming"
        }
        kafka_refined = container "Apache Kafka (Refined Telemetry)" "Imputed/cleaned topics for downstream analytics." "Apache Kafka" {
          tags "EventStreaming"
        }

        social_api = container "Social Ingest API" "Collects webhooks/polls social feeds, normalizes, publishes to Kafka." "HTTP Service (FastAPI/Flask)" {
          tags "Container"
        }

        syslog_api = container "Syslog Ingest API" "Collects RFC3164/RFC5424 syslogs via UDP/TCP; normalizes and publishes to Kafka 'infra.syslog'." "Service (FastAPI/Flask + UDP listener)" {
          tags "Container"
          sys_parser    = component "Syslog Parser" "Parses syslog, extracts ts, host, app, facility, severity, msg." "Parser"
          sys_kafka_sink = component "Kafka Sink (infra.syslog)" "Publishes normalized syslog events." "Kafka Producer"

          sys_parser -> sys_kafka_sink "Serialize to JSON and enqueue"
        }
      }

      group "Stream Processing" {
        spark_predict = container "Apache Spark Streaming ML - Predict Missing Data and anomaly detection" "Detects gaps/out-of-order events and imputes values; writes to refined Kafka and Postgres for TB." "Spark Structured Streaming" {
          tags "StreamJob"
          sp_src    = component "Kafka Source (iot.telemetry, social.events, infra.metrics, infra.syslog)" "Reads JSON telemetry & syslog with event-time, deviceId/host, metrics.*, severity." "spark.readStream.format('kafka')"
          sp_gap    = component "Gap Detector" "Watermark and per-device state." "mapGroupsWithState"
          sp_impute = component "Imputer" "Forward-fill and linear interpolation; flags synthetic values." "UDF/Window"
          sp_sinkk  = component "Kafka Sink (refined.telemetry)" "Publishes cleaned/imputed stream." "spark.writeStream.to('kafka')"
          sp_sankt  = component "PostgreSQL Sink (TB Telemetry)" "Writes cleaned/imputed series for TB publishing." "foreachBatch JDBC"
          sp_src -> sp_gap "Parse and group by deviceId/host"
          sp_gap -> sp_impute "For rows with gaps or missing metrics"
          sp_impute -> sp_sinkk "Publish"
          sp_impute -> sp_sankt "Upsert (ts, deviceId, metric, value, is_imputed)"
        }

        spark_analytical = container "Apache Spark Streaming - Analytical Model to Neo4j" "Consumes refined telemetry for analytics; updates Neo4j and writes insights for TB to Postgres." "Spark Structured Streaming" {
          tags "StreamJob"
        }

        graph_spark = container "Spark Job: Graph QA" "Reads Neo4j, parses JSON, runs GraphFrames, writes KPIs to Postgres for TB." "Spark Structured Streaming" {
          tags "StreamJob"
        }
      }

      group "Storage" {
        tdengine = container "TDengine" "Time-series telemetry and model outputs." "TDengine" {
          tags "Database"
        }
        neo4j = container "Neo4j" "Entities/relationships enriched by analytics." "Neo4j (Bolt/HTTP)" {
          tags "GraphDB"
        }
        pg_stream = container "PostgreSQL (Telemetry for TB)" "Aggregated telemetry/KPIs destined for ThingsBoard." "PostgreSQL" {
          tags "RelationalDB"
        }
        pg_tb = container "PostgreSQL (ThingsBoard)" "Device metadata, relations, alarms." "PostgreSQL" {
          tags "RelationalDB"
        }
      }

      group "Serving & UI" {
        thingsboard = container "ThingsBoard" "Dashboards, rules, alerts." "ThingsBoard (Web App)" {
          tags "WebApp"
        }
        tb_api = container "Telemetry API" "Reads from 'PostgreSQL (Telemetry for TB)' and pushes telemetry/attributes to ThingsBoard." "HTTP Service (FastAPI/Flask)" {
          tags "Container"
        }
      }

      social -> social_api "Webhooks & API pulls" "HTTPS"
      iot -> mqtt "Publish device telemetry" "MQTT"
      sys -> syslog_api "Send syslog messages" "UDP/TCP 514"
      social_api -> kafka_ingest "Publish social.events" "Kafka Producer"
      mqtt -> mqtt_source "Subscribe device/# (e.g., telemetry/+/+)" "MQTT client"
      mqtt_source -> kafka_ingest "Publish iot.telemetry" "Kafka Producer"
      syslog_api -> kafka_ingest "Publish infra.syslog" "Kafka Producer"
      kafka_ingest -> spark_predict "Consume raw topics" "Kafka Consumer"
      spark_predict -> kafka_refined "Publish refined telemetry" "Kafka Producer"
      kafka_refined -> spark_analytical "Consume refined telemetry" "Kafka Consumer"
      spark_predict -> tdengine "Write cleaned/imputed series (for TB)" "JDBC/SQL"
      spark_analytical -> neo4j "Upsert entities/relationships" "Bolt/HTTP"
      neo4j -> graph_spark "Read nodes/edges (Bolt/Cypher)" "Connector"
      graph_spark -> pg_stream "Write graph KPIs/flags (for TB)" "JDBC/SQL"
      tb_api -> pg_stream "Read telemetry/insights to publish" "JDBC/SQL"
      tb_api -> thingsboard "Push telemetry/attributes/alarms" "HTTPS/MQTT"
      tdengine -> thingsboard "Pushes IoT/SCADA data"
      thingsboard -> pg_tb "Store metadata/relations/rule chains/alarms" "JDBC/SQL"
    }

    operator -> thingsboard "View dashboards and manage devices" "HTTPS"
  }

  views {
    container rtsa "streaming-platform" {
      title "Streaming Platform - Container View"
      include *
      
    }

    component spark_predict "spark-predict" {
      title "Spark Predict – Component View"
      include *
      autolayout tb
    }

    component spark_analytical "spark-analytical" {
      title "Spark Analytical – Component View"
      include *
      autolayout tb
    }

    styles {
      element "Person" {
        shape Person
        background #0f766e
        color #ffffff
      }
      element "External" {
        background #9ca3af
        color #ffffff
        opacity 60
      }
      element "DataSource" {
        shape Box
        background #f59e0b
        color #000000
      }
      element "StreamJob" {
        shape RoundedBox
        background #34d399
        color #000000
      }
      element "WebApp" {
        shape WebBrowser
        background #60a5fa
        color #000000
      }
      element "Database" {
        shape Cylinder
        background #c084fc
        color #000000
      }
      element "GraphDB" {
        shape Cylinder
        background #f472b6
        color #000000
      }
      element "RelationalDB" {
        shape Cylinder
        background #93c5fd
        color #000000
      }
      element "Software System" {
        background #1168bd
        color #ffffff
      }
      element "Container" {
        background #438dd5
        color #ffffff
      }
      element "EventStreaming" {
        shape Box
        background #f97316
        color #000000
      }
      element "MessageBroker" {
        shape Box
        background #10b981
        color #000000
      }
      element "Integration" {
        shape Box
        background #8b5cf6
        color #ffffff
      }
      element "Group" {
        background #f3f4f6
        color #111827
        opacity 40
      }
      relationship "Relationship" {
        routing curved
        dashed true
      }
    }
  }
}