application-example.conf

# This is an example Lucille configuration file
# Lucille interprets this file using the Typesafe Config library: https://github.com/lightbend/config
# This file is in HOCON format, a superset of JSON
# In IntelliJ IDEA, HOCON can be rendered and formatted using: https://plugins.jetbrains.com/plugin/10481-hocon

###################
# CONNECTORS: a list of named connectors to be executed in sequence as part of a Lucille "run"
# Each connector reads data from a source and emits that data as a sequence of Documents to be processed by a specific Lucille pipeline
connectors: [
  {
    # name to assign to this connector, for use in logging / console output
    name: "connector1",

    # connector implementation class
    class: "com.kmwllc.lucille.connector.CSVConnector",

    # name of pipeline that will process this connector's output; see Pipeline config block below
    pipeline: "pipeline1",

    # property specific to CSVConnector indicating path to source file
    path: "/Volumes/Work/lucille/src/test/resources/test.csv"
  },
  {
    name: "connector2",
    class: "com.kmwllc.lucille.connector.CSVConnector",
    pipeline: "pipeline2"
    path: "/Volumes/Work/lucille/src/test/resources/test4.csv"
  }
]

###################
# PIPELINES: Each pipeline is a list of Stages that will be applied every incoming Document to prepare it for indexing
# The Pipelines defined below are used by the Connectors defined above; multiple Connectors may feed to the same Pipeline
pipelines: [
  {
    # name to assign to this pipeline
    name: "pipeline1",

    # list of Stages to be applied to each Document flowing through this pipeline
    stages: [

      # first and only stage in this pipeline
      {
        # name to assign to this stage
        name: "copyFields1",

        # stage implementation class
        class: "com.kmwllc.lucille.stage.CopyFields",

        # stage-specific parameter
        source: ["input1", "input2", "input3"],

        # stage-specific parameter
        dest: ["output1", "output2", "output3"],

        # stage-specific parameter
        update_mode: "overwrite"
      }
    ]
  },
  {
    name: "pipeline2",
    stages: [
      {
        class: "com.kmwllc.lucille.stage.CreateChildrenStage"
      }
    ]
  }
]

# Alternative syntax for declaring pipelines, allowing the list of pipelines to be declared and then extended later
pipelines: [{name: "pipeline1", stages: [{class: "com.kmwllc.lucille.stage.CreateChildrenStage"}]}]
pipelines: ${pipelines} [{name: "pipeline2", stages: [{class: "com.kmwllc.lucille.stage.CreateChildrenStage"}]}] # updates previous line

# See also: file-to-file-example.conf, which illustrates how to reference other config files within a config file


##################
# INDEXER: an Indexer sends processed Documents to a search engine or other destination
# At present, only one Indexer can be defined and it will be used for all Pipelines
# This block controls the Indexer type and other general settings
indexer {

  # indexer type (Solr, OpenSearch, Elasticsearch, CSV); if omitted, defaults to Solr
  type: "Solr"

  # indexer implementation class may be provided instead of type; use for plugins and custom implementations
  class: "com.kmwllc.lucille.pinecone.indexer.PineconeIndexer"

  # the number of milliseconds (since the previous add or flush) beyond which the batch will be considered as expired
  # and ready to flush regardless of its size; defaults to 100
  batchTimeout: 6000

  # maximum size of a batch before it is flushed
  batchSize: 100

  # field containing an id that should be sent to the destination index/collection for any given doc,
  # in place of the value of the Document.ID_FIELD field
  idOverrideField: "identification"

  # field containing the name of the index/collection that should be the destination for the given doc, in place of the default index
  indexOverrideField: "new_index_field"

  # fields that should never be included when sending a document to the destination
  ignoreFields: ["ignore_this_field1", "ignore_this_field2"]

  # enable or disable indexing; defaults to true
  # set to false to disable indexing for testing, or when no indexer is required
  sendEnabled: true

  # At present, following settings are only used by the Solr Indexer.
  # When a Lucille document is treated as a deletion request (because it has deletionMarkerField set to the deletionMarkerValue)
  # but it is not a delete-by-query request (because it doesn't have deleteByFieldField and deleteByFieldValue), then
  # it will be treated as a delete-by-id request: the solr document with the same ID as the Lucille document will be deleted.
  # If the Lucille document has a deleteByFieldField and a deleteByFieldValue (in addition to deletionMarkerField set to
  # deletionMarkerValue), it will be treated as a delete-by-query request. All of the documents in solr containing that
  # value in that field will be deleted.

  # these settings determine whether a document is considered as a deletion request
  deletionMarkerField: "is_deleted" # name of field that indicates whether a document represents a deletion request
  deletionMarkerFieldValue: "true" # value that, when present in deletionMarkerField, indicates a deletion request

  # a document field containing the name of field that should be used in a delete-by-query request
  deleteByFieldField: "field_to_delete"
  # a document field containing a value to look for when issuing a delete-by-query request against deleteByFieldField
  deleteByFieldValue: "false"

}

# Search engine (destination) connection info used by the Indexer
# The blocks below show how to provide connection info and other settings for specific search backends
# If the indexer.type is set to "Solr" above then a solr block must be provided, etc.

# Solr basic indexer config (uses HTTP2SolrClient)
solr {
  url: "http://localhost:8983/solr/collection1" # url includes the collection name (i.e. "collection1")
}

# Solr cloud mode with url (uses CloudHTTP2SolrClient)
solr {
  useCloudClient: true
  url: ["http://localhost:8983/solr"] # url should not include collection name
  defaultCollection: "collection2"
}

# Solr cloud mode with zkHosts
solr {
  useCloudClient: true
  zkHosts: ["zookeeper1:2181", "zookeeper2:2181", "zookeeper3:2181"]
  defaultCollection: "collection3"

  # optional
  zkChroot: "/solr"
}

# Elasticsearch
elastic {
  url: "http://localhost:9200"
  index: "index1"
  type: "lucille-type"
  sendEnabled: false
}

# OpenSearch
opensearch {
  url: "https://admin:admin@localhost:9200"
  url: ${?OPENSEARCH_URL} # allow env override
  index: "index2"
  index: ${?OPENSEARCH_INDEX}
  acceptInvalidCert: false # only enable for testing ssl/https against localhost
}

#################
# WORKER: each worker creates an instance of a specific Pipeline and sends Document through that Pipeline
worker {
  # name of the pipeline to execute
  # this setting is only needed when a Worker or WorkerIndexer is run as a separate process
  # (when Lucille runs in "local" mode, the Runner instantiates the Worker with the appropriate pipeline for each Connector,
  # and the pipeline should not be specified here)
  pipeline: "pipeline_name"

  # number of worker threads to start for each pipeline when running lucille in local mode
  threads: 2

  # tells the worker to System.exit(1) assuming that the worker has not polled before the maximum time given
  exitOnTimeout: "true"

  # maximum time to spend processing a message, after which the worker will assume a problem and shut down, assuming that
  # worker.exitOnTimeout=true
  maxProcessingSecs: 600 # 10 minutes

  # maximum number of times across all workers that an attempt should be made to process any given document;
  # when this property is not provided, retries will not be tracked and no limit will be imposed;
  # this property requires that zookeeper is available at the specified zookeeper.connectString
  maxRetries: 2

  # tell the worker process to generate a heartbeat.log that can be used to check liveness
  enableHeartbeat: true

  # time in ms, between successive updates to heartbeat.log
  period: 6000
}

zookeeper {
  # connect string for zookeeper ensemble to use when worker.maxRetries is set
  connectString: "localhost:2181"
}

################
# KAFKA: when Lucille runs in hybrid or distributed mode, components communicate through various Apache Kafka topics
# This section provides Kafka connection and configuration info
kafka {
  bootstrapServers: "localhost:9092"

  # how long a kafka poll should wait for data before returning an empty record set
  pollIntervalMs: 250

  # maximum time allowed between kafka polls before consumer is evicted from consumer group
  maxPollIntervalSecs: 600 # 10 minutes

  # ID of consumer group that all lucille workers should belong to
  consumerGroupId: "lucille_workers"

  maxRequestSize: 250000000

  securityProtocol: "SSL"

  # does not need to be set explicitly, unless you need to use a custom Deserializer
  documentDeserializer: "com.kmwllc.lucille.message.KafkaDocumentDeserializer"

  # does not need to be set explicitly, unless you need to use a custom Serializer
  documentSerializer: "com.kmwllc.lucille.message.KafkaDocumentSerializer"

  # if set to false, will not send Document failures / successes as messages to a Kafka event topic
  events: true

  # can set custom Kafka topic name which contains documents to be processed
  sourceTopic: "pipeline1_source"

  consumerPropertyFile: ".../consumer-conf/consumer.properties"

  producerPropertyFile: ".../consumer-conf/producer.properties"

  adminPropertyFile: ".../admin-conf/admin.properties"
}

##################
# RUNNER: the Runner is the command-line entry point for launching a Lucille run
# The runner launches Connectors, Workers, and an Indexer, waits for the completion of all work, and outputs a "run summary"
runner {
  # outputs detailed metrics to the main lucille log at the end of each run
  metricsLoggingLevel: "INFO"
  # sets the connector timeout (in ms), defaults to 86400000ms
  connectorTimeout: 100000
}

####################
# PUBLISHER: the Publisher is an internal component used by Connectors to send Documents into the system and track their status
publisher {
  # this setting controls
  #     1) maximum size of the queue of published documents waiting to be processed
  #     2) maximum size of the queue of completed documents waiting to be indexed
  # e.g. if it is set to 10 then each each queue can contain up to 10 documents for a total of 20
  #
  # attempts to publish beyond this capacity will cause the publisher to block
  # this setting applies only when running Lucille in local / single-JVM mode using a LocalMessageManager
  # this setting affects Lucille's memory footprint as it determines how many in-flight documents can be held in memory at once
  # this setting defaults to 100
  queueCapacity: 100
}

#########
# MISC

# AWS connection info used by the VFSConnector
aws {
  accessKeyId: "${?AWS_ACCESS_KEY_ID}",
  secretAccessKey: "${?AWS_SECRET_ACCESS_KEY}",
  defaultRegion: "${?AWS_DEFAULT_REGION}"
}

log {
  seconds: 30 # how often components (Publisher, Worker, Indexer) should log a status update
}