EE Refactor

.github/workflows/backend-check.yml

@@ -53,7 +53,7 @@ jobs:
         timeout-minutes: 16
         run: |
           mkdir -p fake_frontend_build
-          FRONTEND_BUILD_DIR=$(pwd)/fake_frontend_build SQLX_OFFLINE=true cargo check --all-features
+          FRONTEND_BUILD_DIR=$(pwd)/fake_frontend_build SQLX_OFFLINE=true cargo check --features $(./all_features_oss.sh)
 
   check_ee:
     runs-on: ubicloud-standard-8

backend/.gitignore

@@ -6,4 +6,5 @@ tracing.folded
 heaptrack*
 index/
 windmill-api/openapi-*.*
-.duckdb/*
+.duckdb/*
+*ee.rs

backend/Cargo.toml

@@ -49,6 +49,7 @@ lto = "thin"
 
 [features]
 default = []
+private = ["windmill-api/private", "windmill-autoscaling/private", "windmill-common/private", "windmill-git-sync/private", "windmill-indexer/private", "windmill-queue/private", "windmill-worker/private"]
 agent_worker_server = ["windmill-api/agent_worker_server"]
 enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise", "windmill-git-sync/enterprise", "windmill-common/prometheus", "windmill-common/enterprise"]
 enterprise_saml = ["windmill-api/enterprise_saml", "oauth2"]
@@ -94,7 +95,7 @@ php = ["windmill-worker/php"]
 csharp = ["windmill-worker/csharp"]
 nu = ["windmill-worker/nu"]
 java = ["windmill-worker/java"]
-all_languages = [ "python", "deno_core", "rust", "mysql", "oracledb", "duckdb", "mssql", "bigquery", "csharp", "nu", "php", "java"]
+all_languages = ["python", "deno_core", "rust", "mysql", "oracledb", "duckdb", "mssql", "bigquery", "csharp", "nu", "php", "java"]
 
 
 [patch.crates-io]

backend/all_features_oss.sh

@@ -0,0 +1,20 @@
+# This script outputs all features except private. Usage :
+#  > cargo build --features $(./all_features_oss.sh)
+
+#!/bin/bash
+
+# Path to the Cargo.toml file
+CARGO_TOML_PATH="./Cargo.toml"
+
+# Extract features from Cargo.toml and output them separated by commas
+if [[ -f "$CARGO_TOML_PATH" ]]; then
+    grep -A 100 '\[features\]' "$CARGO_TOML_PATH" | \
+    sed -n '/\[features\]/,/^\[/p' | \
+    grep -E '^[a-zA-Z0-9_-]+' | \
+    grep -v 'private' | \
+    cut -d' ' -f1 | \
+    paste -sd ',' -
+else
+    echo "Cargo.toml not found at $CARGO_TOML_PATH"
+    exit 1
+fi

backend/ee-repo-ref.txt

@@ -1 +1 @@
-8a2506e86b923c00522cb83b052586f705f7aa8e
+70895a4a8f8891032c5b478a37ab6fafd0d4a9d0

backend/src/ee.rs → backend/src/ee_oss.rs

@@ -1,8 +1,13 @@
+#[cfg(feature = "private")]
+#[allow(unused)]
+pub use crate::ee::*;
+
+#[cfg(not(feature = "private"))]
 pub async fn set_license_key(_license_key: String) -> () {
     // Implementation is not open source
 }
 
-#[cfg(feature = "enterprise")]
+#[cfg(all(feature = "enterprise", not(feature = "private")))]
 pub async fn verify_license_key() -> () {
     // Implementation is not open source
 }

backend/src/main.rs

@@ -28,7 +28,9 @@ use uuid::Uuid;
 use windmill_api::HTTP_CLIENT;
 
 #[cfg(feature = "enterprise")]
-use windmill_common::ee::{maybe_renew_license_key_on_start, LICENSE_KEY_ID, LICENSE_KEY_VALID};
+use windmill_common::ee_oss::{
+    maybe_renew_license_key_on_start, LICENSE_KEY_ID, LICENSE_KEY_VALID,
+};
 
 use windmill_common::{
     agent_workers::build_agent_http_client,
@@ -49,7 +51,7 @@ use windmill_common::{
         TIMEOUT_WAIT_RESULT_SETTING,
     },
     scripts::ScriptLang,
-    stats_ee::schedule_stats,
+    stats_oss::schedule_stats,
     triggers::TriggerKind,
     utils::{hostname, rd_string, Mode, GIT_VERSION, MODE_AND_ADDONS},
     worker::{
@@ -98,7 +100,9 @@ const DEFAULT_NUM_WORKERS: usize = 1;
 const DEFAULT_PORT: u16 = 8000;
 const DEFAULT_SERVER_BIND_ADDR: Ipv4Addr = Ipv4Addr::new(0, 0, 0, 0);
 
-mod ee;
+#[cfg(feature = "private")]
+pub mod ee;
+mod ee_oss;
 mod monitor;
 
 pub fn setup_deno_runtime() -> anyhow::Result<()> {
@@ -552,7 +556,7 @@ Windmill Community Edition {GIT_VERSION}
                 _ = indexer_rx.recv() => {
                     tracing::info!("Received killpill, aborting index initialization");
                 },
-                res = windmill_indexer::completed_runs_ee::init_index(&db) => {
+                res = windmill_indexer::completed_runs_oss::init_index(&db) => {
                         let res = res?;
                         reader = Some(res.0);
                         writer = Some(res.1);
@@ -574,7 +578,7 @@ Windmill Community Edition {GIT_VERSION}
             async {
                 if let Some(db) = conn.as_sql() {
                     if let Some(index_writer) = index_writer2 {
-                        windmill_indexer::completed_runs_ee::run_indexer(
+                        windmill_indexer::completed_runs_oss::run_indexer(
                             db.clone(),
                             index_writer,
                             indexer_rx,
@@ -596,7 +600,7 @@ Windmill Community Edition {GIT_VERSION}
                     _ = indexer_rx.recv() => {
                         tracing::info!("Received killpill, aborting index initialization");
                     },
-                    res = windmill_indexer::service_logs_ee::init_index(&db, killpill_tx.clone()) => {
+                    res = windmill_indexer::service_logs_oss::init_index(&db, killpill_tx.clone()) => {
                             let res = res?;
                             reader = Some(res.0);
                             writer = Some(res.1);
@@ -618,7 +622,7 @@ Windmill Community Edition {GIT_VERSION}
             async {
                 if let Some(db) = conn.as_sql() {
                     if let Some(log_index_writer) = log_index_writer2 {
-                        windmill_indexer::service_logs_ee::run_indexer(
+                        windmill_indexer::service_logs_oss::run_indexer(
                             db.clone(),
                             log_index_writer,
                             log_indexer_rx,
@@ -1086,7 +1090,7 @@ Windmill Community Edition {GIT_VERSION}
                             tracing::info!("Reloading config after 12 hours");
                             initial_load(&conn, tx.clone(), worker_mode, server_mode, #[cfg(feature = "parquet")] disable_s3_store).await;
                             #[cfg(feature = "enterprise")]
-                            ee::verify_license_key().await;
+                            ee_oss::verify_license_key().await;
                         }
                     }
                 },

backend/src/monitor.rs

@@ -29,9 +29,9 @@ use windmill_api::{
 };
 
 #[cfg(feature = "enterprise")]
-use windmill_common::ee::low_disk_alerts;
+use windmill_common::ee_oss::low_disk_alerts;
 #[cfg(feature = "enterprise")]
-use windmill_common::ee::{jobs_waiting_alerts, worker_groups_alerts};
+use windmill_common::ee_oss::{jobs_waiting_alerts, worker_groups_alerts};
 
 use windmill_common::client::AuthedClient;
 #[cfg(feature = "oauth2")]
@@ -41,7 +41,7 @@ use windmill_common::s3_helpers::reload_object_store_setting;
 use windmill_common::{
     agent_workers::DECODED_AGENT_TOKEN,
     auth::create_token_for_owner,
-    ee::CriticalErrorChannel,
+    ee_oss::CriticalErrorChannel,
     error,
     flow_status::{FlowStatus, FlowStatusModule},
     global_settings::{
@@ -87,9 +87,9 @@ use windmill_worker::{
 use windmill_common::s3_helpers::ObjectStoreReload;
 
 #[cfg(feature = "enterprise")]
-use crate::ee::verify_license_key;
+use crate::ee_oss::verify_license_key;
 
-use crate::ee::set_license_key;
+use crate::ee_oss::set_license_key;
 
 #[cfg(feature = "prometheus")]
 lazy_static::lazy_static! {
@@ -1609,7 +1609,7 @@ pub async fn reload_base_url_setting(conn: &Connection) -> error::Result<()> {
 
         if let Some(q) = q_oauth {
             if let Ok(v) = serde_json::from_value::<
-                Option<HashMap<String, windmill_api::oauth2_ee::OAuthClient>>,
+                Option<HashMap<String, windmill_api::oauth2_oss::OAuthClient>>,
             >(q.clone())
             {
                 v
@@ -1630,7 +1630,7 @@ pub async fn reload_base_url_setting(conn: &Connection) -> error::Result<()> {
     {
         if let Some(db) = conn.as_sql() {
             let mut l = windmill_api::OAUTH_CLIENTS.write().await;
-            *l = windmill_api::oauth2_ee::build_oauth_clients(&base_url, oauths, db).await
+            *l = windmill_api::oauth2_oss::build_oauth_clients(&base_url, oauths, db).await
             .map_err(|e| tracing::error!("Error building oauth clients (is the oauth.json mounted and in correct format? Use '{}' as minimal oauth.json): {}", "{}", e))
             .unwrap();
         }

backend/substitute_ee_code.sh

@@ -4,7 +4,6 @@ script_dirpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 root_dirpath="$(cd "${script_dirpath}/.." && pwd)"
 
 REVERT="NO"
-REVERT_PREVIOUS="NO"
 COPY="NO"
 EE_CODE_DIR="../windmill-ee-private/"
 
@@ -18,13 +17,6 @@ while [[ $# -gt 0 ]]; do
       REVERT="YES"
       shift
       ;;
-    --revert-previous)
-      # This is a special case of --revert that will revert to the previous commit.
-      REVERT="YES"
-      REVERT_PREVIOUS="YES"
-      echo "Reverting to previous commit"
-      shift
-      ;;
     -c|--copy)
       # By default, EE files are symlinked. Pass this option to do a real copy instead.
       # This might be necessary if you want to build the Docker Image as Docker COPY seems
@@ -70,29 +62,19 @@ if [ "$REVERT" == "YES" ]; then
   for ee_file in $(find ${EE_CODE_DIR} -name "*ee.rs"); do
     ce_file="${ee_file/${EE_CODE_DIR}/}"
     ce_file="${root_dirpath}/backend/${ce_file}"
-    if [ "$REVERT_PREVIOUS" == "YES" ]; then
-      git checkout HEAD@{3} ${ce_file} || true
-    else
-      git restore --staged ${ce_file} || true
-      git restore ${ce_file} || true
-    fi
+    rm ${ce_file}
   done
 else
   # This replaces all files in current repo with alternative EE files in windmill-ee-private
   for ee_file in $(find "${EE_CODE_DIR}" -name "*ee.rs"); do
-    ce_file="${ee_file/${EE_CODE_DIR}/}"
-    ce_file="${root_dirpath}/backend/${ce_file}"
-    if [[ -f "${ce_file}" ]]; then
-      rm "${ce_file}"
-      if [ "$COPY" == "YES" ]; then
-        cp "${ee_file}" "${ce_file}"
-        echo "File copied '${ee_file}' -->> '${ce_file}'"
-      else
-        ln -s "${ee_file}" "${ce_file}"
-        echo "Symlink created '${ee_file}' -->> '${ce_file}'"
-      fi
+  ce_file="${ee_file/${EE_CODE_DIR}/}"
+  ce_file="${root_dirpath}/backend/${ce_file}"
+    if [ "$COPY" == "YES" ]; then
+      cp "${ee_file}" "${ce_file}"
+      echo "File copied '${ee_file}' -->> '${ce_file}'"
     else
-      echo "File ${ce_file} is not a file, ignoring"
+      ln -s "${ee_file}" "${ce_file}"
+      echo "Symlink created '${ee_file}' -->> '${ce_file}'"
     fi
   done
 fi

backend/update_sqlx.sh

@@ -9,7 +9,7 @@ if [[ "$(uname)" == "Darwin" ]]; then
     sed -i '' 's/^# \(samael = { git="https:\/\/github.com\/njaremko\/samael", rev="464d015e3ae393e4b5dd00b4d6baa1b617de0dd6", features = \["xmlsec"\] }\)/\1/' Cargo.toml
 fi
 
-cargo sqlx prepare --workspace -- --all-targets --all-features
+cargo sqlx prepare --workspace -- --all-targets --features $(./all_features_oss.sh)
 ./substitute_ee_code.sh -r --dir ../windmill-ee-private
 
 # Undo the samael changes on macOS

backend/windmill-api/Cargo.toml

@@ -10,6 +10,7 @@ path = "src/lib.rs"
 
 [features]
 default = []
+private = ["windmill-audit/private"]
 enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker/enterprise"]
 stripe = []
 agent_worker_server = []

backend/windmill-api/src/agent_workers_ee.rs → backend/windmill-api/src/agent_workers_oss.rs

@@ -1,3 +1,7 @@
+#[cfg(feature = "private")]
+#[allow(unused)]
+pub use crate::agent_workers_ee::*;
+
 /*
  * Author: Ruben Fiszel
  * Copyright: Windmill Labs, Inc 2042
@@ -6,16 +10,21 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
+#[cfg(not(feature = "private"))]
 use crate::db::DB;
 
+#[cfg(not(feature = "private"))]
 use axum::Router;
 
+#[cfg(not(feature = "private"))]
 use serde::{Deserialize, Serialize};
 
+#[cfg(not(feature = "private"))]
 pub fn global_service() -> Router {
     Router::new()
 }
 
+#[cfg(not(feature = "private"))]
 pub fn workspaced_service(
     db: DB,
     _base_internal_url: String,
@@ -36,15 +45,18 @@ pub fn workspaced_service(
 }
 
 #[derive(Clone, Debug, Deserialize, Serialize)]
+#[cfg(not(feature = "private"))]
 pub struct AgentAuth {
     pub worker_group: String,
     pub suffix: Option<String>,
     pub tags: Vec<String>,
     pub exp: Option<usize>,
 }
 
+#[cfg(not(feature = "private"))]
 pub struct AgentCache {}
 
+#[cfg(not(feature = "private"))]
 impl AgentCache {
     pub fn new() -> Self {
         AgentCache {}

backend/windmill-api/src/ai.rs

@@ -10,7 +10,7 @@ use reqwest::{Client, RequestBuilder};
 use serde::{Deserialize, Serialize};
 use serde_json::value::RawValue;
 use std::collections::HashMap;
-use windmill_audit::{audit_ee::audit_log, ActionKind};
+use windmill_audit::{audit_oss::audit_log, ActionKind};
 use windmill_common::error::{to_anyhow, Error, Result};
 
 lazy_static::lazy_static! {

backend/windmill-api/src/apps.rs

@@ -18,7 +18,7 @@ use crate::{
 };
 #[cfg(feature = "parquet")]
 use crate::{
-    job_helpers_ee::{
+    job_helpers_oss::{
         download_s3_file_internal, get_random_file_name, get_s3_resource,
         get_workspace_s3_resource, upload_file_from_req, DownloadFileQuery,
     },
@@ -48,7 +48,7 @@ use sha2::{Digest, Sha256};
 use sql_builder::{bind::Bind, SqlBuilder};
 use sqlx::{types::Uuid, FromRow};
 use std::str;
-use windmill_audit::audit_ee::audit_log;
+use windmill_audit::audit_oss::audit_log;
 use windmill_audit::ActionKind;
 use windmill_common::{
     apps::{AppScriptId, ListAppQuery},

backend/windmill-api/src/apps_oss.rs

@@ -0,0 +1,11 @@
+#[cfg(feature = "private")]
+#[allow(unused)]
+pub use crate::apps_ee::*;
+
+#[cfg(not(feature = "private"))]
+use axum::Router;
+
+#[cfg(not(feature = "private"))]
+pub fn global_unauthed_service() -> Router {
+    Router::new()
+}

backend/windmill-api/src/args.rs

@@ -85,7 +85,7 @@ impl RawWebhookArgs {
         db: &DB,
         w_id: &str,
     ) -> Result<HashMap<String, Box<RawValue>>, Error> {
-        use crate::job_helpers_ee::{
+        use crate::job_helpers_oss::{
             get_random_file_name, get_workspace_s3_resource, upload_file_internal,
         };
         use futures::TryStreamExt;

backend/windmill-api/src/audit.rs

@@ -28,7 +28,7 @@ async fn get_audit(
     Path((w_id, id)): Path<(String, i32)>,
 ) -> JsonResult<AuditLog> {
     let tx = user_db.begin(&authed).await?;
-    let audit = windmill_audit::audit_ee::get_audit(tx, id, &w_id).await?;
+    let audit = windmill_audit::audit_oss::get_audit(tx, id, &w_id).await?;
     Ok(Json(audit))
 }
 async fn list_audit(
@@ -39,6 +39,6 @@ async fn list_audit(
     Query(lq): Query<ListAuditLogQuery>,
 ) -> JsonResult<Vec<AuditLog>> {
     let tx = user_db.begin(&authed).await?;
-    let rows = windmill_audit::audit_ee::list_audit(tx, w_id, pagination, lq).await?;
+    let rows = windmill_audit::audit_oss::list_audit(tx, w_id, pagination, lq).await?;
     Ok(Json(rows))
 }