diff --git a/installing_deps.sh b/installing_deps.sh index 6a705c52..a79152f9 100755 --- a/installing_deps.sh +++ b/installing_deps.sh @@ -8,6 +8,7 @@ # SKIP_KVROCKS=1 # SKIP_GEN_CERT=1 # SKIP_DB_SETUP=1 +#. SKIP_LNX_PKG_INSTALL=1 # # Example: SKIP_REDIS=1 SKIP_YARA=1 ./install.sh # @@ -18,6 +19,7 @@ set -e ## bash debug mode toggle below #set -x +if [ -z "$SKIP_LNX_PKG_INSTALL" ]; then sudo apt-get update sudo apt-get install python3-pip virtualenv python3-dev python3-tk libfreetype6-dev \ @@ -54,8 +56,16 @@ sudo apt-get install build-essential libffi-dev autoconf -qq # sflock, gz requirement sudo apt-get install p7zip-full -qq # TODO REMOVE ME +else + echo "--- Skipping Linux packages installation ---" +fi + # SUBMODULES # +if [ -z "$SKIP_GIT_SUBMODULE" ]; then git submodule update --init --recursive +else + echo "--- Skipping Linux packages installation ---" +fi # REDIS # if [ -z "$SKIP_REDIS" ]; then diff --git a/other_installers/docker/.dockerignore b/other_installers/docker-legacy/.dockerignore similarity index 100% rename from other_installers/docker/.dockerignore rename to other_installers/docker-legacy/.dockerignore diff --git a/other_installers/docker-legacy/Dockerfile b/other_installers/docker-legacy/Dockerfile new file mode 100644 index 00000000..3f63372b --- /dev/null +++ b/other_installers/docker-legacy/Dockerfile @@ -0,0 +1,40 @@ +FROM ubuntu:22.04 +ARG tz_buildtime=Europe/Rome +ENV TZ=$tz_buildtime +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +# Make sure that all updates are in place +RUN apt-get clean && apt-get update -y && apt-get upgrade -y \ + && apt-get dist-upgrade -y && apt-get autoremove -y + +# Install needed packages +RUN apt-get install git python3-dev build-essential \ + libffi-dev libssl-dev libfuzzy-dev wget sudo -y + +# Adding sudo command +RUN useradd -m docker && echo "docker:docker" | chpasswd && adduser docker sudo +RUN echo "root ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers + +# Installing AIL dependencies +RUN mkdir /opt/AIL +ADD . /opt/AIL +WORKDIR /opt/AIL +RUN ./installing_deps.sh +WORKDIR /opt/AIL + +# Default to UTF-8 file.encoding +ENV LANG C.UTF-8 +ENV AIL_HOME /opt/AIL +ENV AIL_BIN ${AIL_HOME}/bin +ENV AIL_FLASK ${AIL_HOME}/var/www +ENV AIL_REDIS ${AIL_HOME}/redis/src +ENV AIL_ARDB ${AIL_HOME}/ardb/src +ENV AIL_VENV ${AIL_HOME}/AILENV + +ENV PATH ${AIL_VENV}/bin:${AIL_HOME}:${AIL_REDIS}:${AIL_ARDB}:${AIL_BIN}:${AIL_FLASK}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + +RUN ./pystemon/install.sh +RUN pip install -r /opt/pystemon/requirements.txt + +COPY docker_start.sh /docker_start.sh +ENTRYPOINT ["/bin/bash", "docker_start.sh"] diff --git a/other_installers/docker-legacy/README.md b/other_installers/docker-legacy/README.md new file mode 100644 index 00000000..018ba146 --- /dev/null +++ b/other_installers/docker-legacy/README.md @@ -0,0 +1,45 @@ +Docker Quick Start (Ubuntu 18.04 LTS) +------------ + +:warning: +This Docker is not maintained at the moment. +If you are interested to contribute, please submit a Pull Request + + +1. Install Docker +```bash +sudo su +apt-get install -y curl +curl https://get.docker.com | /bin/bash +``` + +2. Type these commands to build the Docker image: +```bash +git clone https://github.com/ail-project/ail-framework.git +cd AIL-framework +cp -r ./other_installers/docker/Dockerfile ./other_installers/docker/docker_start.sh ./other_installers/docker/pystemon ./ +cp ./configs/update.cfg.sample ./configs/update.cfg +vim/nano ./configs/update.cfg (set auto_update to False) +docker build --build-arg tz_buildtime=YOUR_GEO_AREA/YOUR_CITY -t ail-framework . +``` +3. To start AIL on port 7000, type the following command below: +``` +docker run -p 7000:7000 ail-framework +``` + +4. To debug the running container, type the following command and note the container name or identifier: +```bash +docker ps +``` + +After getting the name or identifier type the following commands: +```bash +docker exec -it CONTAINER_NAME_OR_IDENTIFIER bash +cd /opt/ail +``` + +Install using Ansible +--------------------- + +Please check the [Ansible readme](ansible/README.md). + diff --git a/other_installers/docker-legacy/docker-compose.yml b/other_installers/docker-legacy/docker-compose.yml new file mode 100644 index 00000000..b10b7afd --- /dev/null +++ b/other_installers/docker-legacy/docker-compose.yml @@ -0,0 +1,523 @@ +version: '3' +services: + ardb: + entrypoint: + - ardb-server + - /opt/AIL/configs/6382.conf + healthcheck: + test: ["CMD", "redis-cli", "-p", "6382", "ping"] + interval: 30s + timeout: 10s + retries: 5 + network_mode: service:flask + image: ail-framework + volumes: + - ./configs:/opt/AIL/configs:ro + crawler: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/Crawler.py + - onion + - "8050" + network_mode: service:flask + image: ail-framework + volumes: + - ./CRAWLED_SCREENSHOT/:/opt/AIL/CRAWLED_SCREENSHOT + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + flask: + build: . + entrypoint: + - /opt/AIL/var/www/Flask_server.py + ports: + - "7000:7000" + image: ail-framework + volumes: + - ./CRAWLED_SCREENSHOT/:/opt/AIL/CRAWLED_SCREENSHOT + - ./PASTES/:/opt/AIL/PASTES + - ./indexdir:/opt/AIL/indexdir + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/var/www + log-queue: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/AILENV/bin/log_subscriber + - -p + - "6380" + - -c + - Queing + - -l + - /opt/AIL/logs/ + network_mode: service:flask + image: ail-framework + volumes: + - ./configs:/opt/AIL/configs:ro + log-script: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/AILENV/bin/log_subscriber + - -p + - "6380" + - -c + - Script + - -l + - /opt/AIL/logs/ + network_mode: service:flask + image: ail-framework + volumes: + - ./configs:/opt/AIL/configs:ro + pystemon: + depends_on: + - redis-log + entrypoint: + - /opt/pystemon/pystemon.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./pystemon/archives:/opt/pystemon/archive + - ./pystemon/proxies.txt:/opt/pystemon/proxies.txt:ro + - ./pystemon/pystemon.yaml:/opt/pystemon/pystemon.yaml:ro + working_dir: /opt/pystemon + pystemon-feeder: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/feeder/pystemon-feeder.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./pystemon/archives:/opt/pystemon/archive + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + queues: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/launch_queues.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + redis-cache: + entrypoint: + - redis-server + - /opt/AIL/configs/6379.conf + healthcheck: + test: ["CMD", "redis-cli", "-p", "6379", "ping"] + interval: 30s + timeout: 10s + retries: 5 + image: ail-framework + network_mode: service:flask + volumes: + - ./configs:/opt/AIL/configs:ro + redis-log: + entrypoint: + - redis-server + - /opt/AIL/configs/6380.conf + healthcheck: + test: ["CMD", "redis-cli", "-p", "6380", "ping"] + interval: 30s + timeout: 10s + retries: 5 + network_mode: service:flask + image: ail-framework + volumes: + - ./configs:/opt/AIL/configs:ro + redis-mixer-cache: + entrypoint: + - redis-server + - /opt/AIL/configs/6381.conf + healthcheck: + test: ["CMD", "redis-cli", "-p", "6381", "ping"] + interval: 30s + timeout: 10s + retries: 5 + image: ail-framework + network_mode: service:flask + volumes: + - ./configs:/opt/AIL/configs:ro + script-alerthandler: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/alertHandler.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-apikey: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/ApiKey.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-bankaccount: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/BankAccount.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-bitcoin: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/Bitcoin.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-categ: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/Categ.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-credential: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/Credential.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-creditcards: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/CreditCards.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-curve: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/Curve.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-curvemanagetopsets: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/CurveManageTopSets.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-cve: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/CveModule.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-decoder: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/Decoder.py + network_mode: service:flask + image: ail-framework + volumes: + - ./HASHS:/opt/AIL/HASHS + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-domclassifier: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/DomClassifier.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-duplicates: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/Duplicates.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-global: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/Global.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-indexer: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/Indexer.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./indexdir:/opt/AIL/indexdir + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-keys: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/Keys.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-libinjection: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/LibInjection.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-lines: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/Lines.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-mail: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/Mail.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-mispthehivefeeder: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/MISP_The_Hive_feeder.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-mixer: + depends_on: + - redis-mixer-cache + entrypoint: + - /opt/AIL/bin/Mixer.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-modulestats: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/ModuleStats.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-onion: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/Onion.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-phone: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/Phone.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-regexfortermsfrequency: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/RegexForTermsFrequency.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-release: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/Release.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-sentimentanalysis: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/SentimentAnalysis.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-setfortermsfrequency: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/SetForTermsFrequency.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-sqlinjectiondetection: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/SQLInjectionDetection.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-submitpaste: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/submit_paste.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-tags: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/Tags.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-tokenize: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/Tokenize.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-web: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/Web.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin + script-webstats: + depends_on: + - redis-log + entrypoint: + - /opt/AIL/bin/WebStats.py + network_mode: service:flask + image: ail-framework + volumes: + - ./PASTES/:/opt/AIL/PASTES + - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro + working_dir: /opt/AIL/bin diff --git a/other_installers/docker-legacy/docker_start.sh b/other_installers/docker-legacy/docker_start.sh new file mode 100644 index 00000000..5b0e1b06 --- /dev/null +++ b/other_installers/docker-legacy/docker_start.sh @@ -0,0 +1,21 @@ +#!/bin/bash +signalListener() { + "$@" & + pid="$!" + trap "echo 'Stopping'; kill -SIGTERM $pid" SIGINT SIGTERM + + while kill -0 $pid > /dev/null 2>&1; do + wait + done +} + + +source ./AILENV/bin/activate +cd bin +./LAUNCH.sh -l +./LAUNCH.sh -c +./LAUNCH.sh -f + +signalListener tail -f /dev/null $@ + +./LAUNCH.sh -k diff --git a/other_installers/docker/pystemon/config.cfg b/other_installers/docker-legacy/pystemon/config.cfg similarity index 100% rename from other_installers/docker/pystemon/config.cfg rename to other_installers/docker-legacy/pystemon/config.cfg diff --git a/other_installers/docker/pystemon/install.sh b/other_installers/docker-legacy/pystemon/install.sh old mode 100755 new mode 100644 similarity index 100% rename from other_installers/docker/pystemon/install.sh rename to other_installers/docker-legacy/pystemon/install.sh diff --git a/other_installers/docker/pystemon/proxies.txt b/other_installers/docker-legacy/pystemon/proxies.txt similarity index 100% rename from other_installers/docker/pystemon/proxies.txt rename to other_installers/docker-legacy/pystemon/proxies.txt diff --git a/other_installers/docker/pystemon/pystemon.yaml b/other_installers/docker-legacy/pystemon/pystemon.yaml similarity index 100% rename from other_installers/docker/pystemon/pystemon.yaml rename to other_installers/docker-legacy/pystemon/pystemon.yaml diff --git a/other_installers/docker/Dockerfile b/other_installers/docker/Dockerfile index 3f63372b..8ffc9254 100644 --- a/other_installers/docker/Dockerfile +++ b/other_installers/docker/Dockerfile @@ -1,40 +1,139 @@ -FROM ubuntu:22.04 -ARG tz_buildtime=Europe/Rome -ENV TZ=$tz_buildtime -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone - -# Make sure that all updates are in place -RUN apt-get clean && apt-get update -y && apt-get upgrade -y \ - && apt-get dist-upgrade -y && apt-get autoremove -y +ARG BASE_IMAGE=ubuntu:24.04 +FROM $BASE_IMAGE -# Install needed packages -RUN apt-get install git python3-dev build-essential \ - libffi-dev libssl-dev libfuzzy-dev wget sudo -y +# Installing_deps.sh script arguments +ARG SKIP_REDIS +ARG SKIP_TLSH +ARG SKIP_PGPDUMP +ARG SKIP_YARA +ARG SKIP_KVROCKS +ARG SKIP_GEN_CERT +ARG SKIP_DB_SETUP +ARG SKIP_LNX_PKG_INSTALL +ARG SKIP_GIT_SUBMODULE -# Adding sudo command -RUN useradd -m docker && echo "docker:docker" | chpasswd && adduser docker sudo -RUN echo "root ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers +# Installing_deps.sh script env (arg to env mapping) +ENV SKIP_REDIS=$SKIP_REDIS +ENV SKIP_TLSH=$SKIP_TLSH +ENV SKIP_PGPDUMP=$SKIP_PGPDUMP +ENV SKIP_YARA=$SKIP_YARA +ENV SKIP_KVROCKS=$SKIP_KVROCKS +ENV SKIP_GEN_CERT=$SKIP_GEN_CERT +ENV SKIP_DB_SETUP=$SKIP_DB_SETUP +ENV SKIP_LNX_PKG_INSTALL=$SKIP_LNX_PKG_INSTALL +ENV SKIP_GIT_SUBMODULE=$SKIP_GIT_SUBMODULE -# Installing AIL dependencies -RUN mkdir /opt/AIL -ADD . /opt/AIL -WORKDIR /opt/AIL -RUN ./installing_deps.sh -WORKDIR /opt/AIL - -# Default to UTF-8 file.encoding +# AIL runtime env variables ENV LANG C.UTF-8 -ENV AIL_HOME /opt/AIL +ENV AIL_HOME /home/ail/ail-framework ENV AIL_BIN ${AIL_HOME}/bin ENV AIL_FLASK ${AIL_HOME}/var/www ENV AIL_REDIS ${AIL_HOME}/redis/src ENV AIL_ARDB ${AIL_HOME}/ardb/src ENV AIL_VENV ${AIL_HOME}/AILENV -ENV PATH ${AIL_VENV}/bin:${AIL_HOME}:${AIL_REDIS}:${AIL_ARDB}:${AIL_BIN}:${AIL_FLASK}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin -RUN ./pystemon/install.sh -RUN pip install -r /opt/pystemon/requirements.txt +ARG tz_buildtime=Europe/Prague +ENV TZ=$tz_buildtime +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +#ENV PATH ${AIL_VENV}/bin:${AIL_HOME}:${AIL_REDIS}:${AIL_ARDB}:${AIL_BIN}:${AIL_FLASK}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + +# Build Dependencies +RUN apt-get update && \ + apt-get install -y \ + sudo \ + wget \ + git \ + python3-dev \ + g++ \ + cmake \ + automake \ + libtool \ + make \ + gcc \ + pkg-config \ + build-essential \ + autoconf \ + virtualenv \ + unzip \ + libsnappy-dev \ + libssl-dev \ + libfreetype6-dev \ + protobuf-compiler \ + libprotobuf-dev \ + libadns1-dev \ + libev-dev \ + libgmp-dev \ + libfuzzy-dev \ + libffi-dev && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# Runtime Dependencies +RUN apt-get update && \ + apt-get install -y \ + python3-pip \ + python3-tk \ + screen \ + python3-numpy \ + python3-opencv \ + libzbar0 \ + libadns1 \ + graphviz \ + p7zip-full && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# Replacements of Comiles/Builds +RUN wget https://github.com/RocksLabs/kvrocks-fpm/releases/download/202502091/kvrocks_2.11.1-1_amd64.deb && \ + apt-get update && \ + apt-get install -y \ + pgpdump \ + tlsh-tools \ + yara \ + ./kvrocks_2.11.1-1_amd64.deb && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# Create ail user +RUN useradd -m -s /bin/bash ail + +# Create ail framework work directory +RUN mkdir -p ${AIL_HOME} + +# Change to ail work directory +WORKDIR ${AIL_HOME} + +# Copy AIL dirs +COPY --parents bin configs doc files logs samples tests tools update var ./ + +# Copy AIL files +COPY install_virtualenv.sh installing_deps.sh requirements.txt reset_AIL.sh ./ + +# Copy Docker modified files part 1 - to pass installing_deps.sh check +COPY other_installers/docker/core.cfg configs/core.cfg + +# Start AIL installers +RUN apt-get update && \ + chmod +x ./installing_deps.sh && \ + sed -i 's/^sudo *//' ./installing_deps.sh && \ + ./installing_deps.sh && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# Copy Docker modified files part 2 +COPY other_installers/docker/update.cfg configs/update.cfg + +COPY other_installers/docker/kvrocks_6383.conf configs/6383.conf + +COPY other_installers/docker/docker_start.sh /usr/local/sbin/ + +RUN chmod +x /usr/local/sbin/docker_start.sh + +RUN chown -R ail:ail /home/ail/ + +# Change user +USER ail -COPY docker_start.sh /docker_start.sh -ENTRYPOINT ["/bin/bash", "docker_start.sh"] +ENTRYPOINT ["/bin/bash", "/usr/local/sbin/docker_start.sh"] \ No newline at end of file diff --git a/other_installers/docker/README.md b/other_installers/docker/README.md index 018ba146..8587a803 100644 --- a/other_installers/docker/README.md +++ b/other_installers/docker/README.md @@ -1,45 +1,115 @@ -Docker Quick Start (Ubuntu 18.04 LTS) ------------- +# AIL Framework Docker Installation -:warning: -This Docker is not maintained at the moment. -If you are interested to contribute, please submit a Pull Request +This document provides instructions on how to build and run the AIL framework using Docker or Podman with `docker-compose` or `podman-compose`. +The provided setup is designed to run AIL in a multi-container environment, with separate containers for the AIL application, Valkey instances for caching and queues, and a Kvrocks instance for persistent storage. -1. Install Docker -```bash -sudo su -apt-get install -y curl -curl https://get.docker.com | /bin/bash -``` +## Prerequisites + +- [Docker](https://docs.docker.com/get-docker/) or [Podman](https://podman.io/getting-started/installation) +- `docker-compose` or `podman-compose` + +## Build the AIL Container Image + +First, clone the AIL framework repository and initialize the submodules: -2. Type these commands to build the Docker image: ```bash git clone https://github.com/ail-project/ail-framework.git -cd AIL-framework -cp -r ./other_installers/docker/Dockerfile ./other_installers/docker/docker_start.sh ./other_installers/docker/pystemon ./ -cp ./configs/update.cfg.sample ./configs/update.cfg -vim/nano ./configs/update.cfg (set auto_update to False) -docker build --build-arg tz_buildtime=YOUR_GEO_AREA/YOUR_CITY -t ail-framework . +cd ail-framework +git submodule update --init --recursive ``` -3. To start AIL on port 7000, type the following command below: + +Next, build the AIL container image using the provided `Dockerfile`. The following command builds an image named `localhost/ail`, which is the default image used in the `docker-compose.yml` file. + +```bash +podman build -t localhost/ail \ + --build-arg "BASE_IMAGE=ubuntu:24.04" \ + --build-arg "SKIP_REDIS=1" \ + --build-arg "SKIP_PGPDUMP=1" \ + --build-arg "SKIP_YARA=1" \ + --build-arg "SKIP_KVROCKS=1" \ + --build-arg "SKIP_GEN_CERT=1" \ + --build-arg "SKIP_DB_SETUP=1" \ + --build-arg "SKIP_LNX_PKG_INSTALL=1" \ + --build-arg "SKIP_GIT_SUBMODULE=1" \ + -f other_installers/docker/Dockerfile . ``` -docker run -p 7000:7000 ail-framework + +**Note on Build Arguments:** The `SKIP_*` build arguments are used to disable the installation of dependencies that are provided by other containers in the compose setup (like Redis/Valkey and Kvrocks). This keeps the AIL container image smaller and more focused on the application itself. + +## Running the AIL Framework + +The recommended way to run the AIL framework is by using `podman-compose` (or `docker-compose`) with the provided `docker-compose.yml` file. + +### 1. Launch the Services + +Navigate to the `other_installers/docker` directory and start all the services in the background: + +```bash +cd other_installers/docker +podman-compose up -d ``` -4. To debug the running container, type the following command and note the container name or identifier: +This will start the following services: +- `ail`: The main AIL application container. +- `kvrocks`: The Kvrocks database for persistent storage. +- `cache`, `log`, `log-submit`, `queues`, `process`, `mixer-cache`: Six separate Valkey instances for various caching and queuing purposes. + +### 2. Create a Default User + +After the containers have started, you need to create an initial user to log in to the web interface. + ```bash -docker ps +podman exec -it ail /bin/bash -c ". ./AILENV/bin/activate && cd var/www && python3 ./create_default_user.py" ``` -After getting the name or identifier type the following commands: +Follow the prompts to create the user. + +### 3. Accessing AIL + +The AIL web interface will be available at `https://localhost:7000/`. + +### Managing the Services + +- **View Logs**: To view the logs of all running services, use: + ```bash + podman-compose logs -f + ``` + To view the logs of a specific service (e.g., `ail`): + ```bash + podman-compose logs -f ail + ``` + +- **Stop Services**: To stop and remove the containers, networks, and volumes, use: + ```bash + podman-compose down + ``` + +### Using Custom Images + +You can override the default Valkey and Kvrocks images by setting environment variables before running `podman-compose up`: + ```bash -docker exec -it CONTAINER_NAME_OR_IDENTIFIER bash -cd /opt/ail +KVROCKS_IMAGE=apache/kvrocks:2.14.0 VALKEY_IMAGE=valkey/valkey:8 podman-compose up -d ``` -Install using Ansible ---------------------- +## Configuration + +The AIL framework can be configured by editing the `core.cfg` file located in the `other_installers/docker` directory. This file is mounted into the `ail` container and will be applied on startup. + +## Running the AIL Container Standalone (for Debugging) -Please check the [Ansible readme](ansible/README.md). +It is also possible to run the AIL container by itself, but this is mainly intended for debugging and development. When running standalone, you will need to ensure that the AIL container can connect to externally running Redis/Valkey and Kvrocks instances. + +Here is a basic example of how to run the AIL container standalone: + +```bash +podman run --rm -p 7000:7000 --name ail \ + -e SKIP_LAUNCH_REDIS=true \ + -e SKIP_LAUNCH_KVROCKS=true \ + -e SKIP_CHECK_REDIS=true \ + -e SKIP_CHECK_KVROCKS=true \ + localhost/ail +``` +In this mode, you would need to modify `core.cfg` to point to your database instances. diff --git a/other_installers/docker/core.cfg b/other_installers/docker/core.cfg new file mode 100644 index 00000000..e8f3df25 --- /dev/null +++ b/other_installers/docker/core.cfg @@ -0,0 +1,309 @@ +[Directories] +bloomfilters = Blooms +dicofilters = Dicos +pastes = PASTES +hash = HASHS +crawled = crawled +har = CRAWLED_SCREENSHOT +screenshot = CRAWLED_SCREENSHOT/screenshot +images = IMAGES +favicons = FAVICONS + +wordtrending_csv = var/www/static/csv/wordstrendingdata +wordsfile = files/wordfile + +protocolstrending_csv = var/www/static/csv/protocolstrendingdata +protocolsfile = files/protocolsfile + +tldstrending_csv = var/www/static/csv/tldstrendingdata +tldsfile = faup/src/data/mozilla.tlds + +domainstrending_csv = var/www/static/csv/domainstrendingdata + +sentiment_lexicon_file = sentiment/vader_lexicon.zip/vader_lexicon/vader_lexicon.txt + +[Pystemon] +dir = /home/pystemon/pystemon/ +redis_host = cache +redis_port = 6379 +redis_db = 10 + +##### Logs ###### +[Logs] +# activate syslog +ail_logs_syslog = False +ail_logs_syslog_server = +# default=514 +ail_logs_syslog_port = +# ['auth', 'authpriv', 'cron', 'daemon', 'ftp', 'kern', 'lpr', 'mail', 'news', 'syslog', 'user', 'uucp', 'local0', 'local1', 'local2', 'local3', 'local4', 'local5', 'local6', 'local7'] +ail_logs_syslog_facility = +# ['DEBUG', 'INFO', 'NOTICE', 'WARNING', 'ERROR', 'CRITICAL'] +ail_logs_syslog_level = + +##### Notifications ###### +[Notifications] +ail_domain = https://localhost:7000 +sender = sender@example.com +sender_host = smtp.example.com +sender_port = 1337 +sender_pw = None +# Only needed for SMTP over SSL if the mail server don't support TLS (used by default). use this option to validate the server certificate. +cert_required = False +# Only needed for SMTP over SSL if you want to validate your self signed certificate for SSL +ca_file = +# Only needed when the credentials for email server needs a username instead of an email address +#sender_user = sender +sender_user = + +# optional for using with authenticated SMTP over SSL +# sender_pw = securepassword + +##### Flask ##### +[Flask] +#Proxying requests to the app +baseUrl = / +#Host to bind to +host = 0.0.0.0 +#Flask server port +port = 7000 +#Number of logs to display in the dashboard +max_dashboard_logs = 15 +#Maximum number of character to display in the toolip +max_preview_char = 250 +#Maximum number of character to display in the modal +max_preview_modal = 800 +#Default number of header to display in trending graphs +default_display = 10 +#Number of minutes displayed for the number of processed pastes. +minute_processed_paste = 10 +#Maximum line length authorized to make a diff between duplicates +DiffMaxLineLength = 10000 + +##### Images ##### +[Images] +ollama_url = http://127.0.0.1:11434 +ollama_enabled = True + +##### Users ##### +[Users] +force_2fa = False +2fa_name = AIL + +[AIL_2_AIL] +server_host = 0.0.0.0 +server_port = 4443 +local_addr = + +#### Modules #### +[BankAccount] +max_execution_time = 60 + +[Categ] +#Minimum number of match between the paste and the category file +matchingThreshold=1 + +[Credential] +#Minimum length that a credential must have to be considered as such +minimumLengthThreshold=3 +#Will be pushed as alert if the number of credentials is greater to that number +criticalNumberToAlert=8 +#Will be considered as false positive if less that X matches from the top password list +minTopPassList=5 + +[Decoder] +max_execution_time_base64 = 60 +max_execution_time_binary = 60 +max_execution_time_hexadecimal = 60 + +[Onion] +save_i2p = False +max_execution_time = 180 + +[PgpDump] +max_execution_time = 60 + +[Modules_Duplicates] +#Number of month to look back +maximum_month_range = 3 +#The value where two pastes are considerate duplicate for ssdeep. +threshold_duplicate_ssdeep = 50 +#The value where two pastes are considerate duplicate for tlsh. +threshold_duplicate_tlsh = 52 +#Minimum size of the paste considered +min_paste_size = 0.3 + +[Module_ModuleInformation] +#Threshold to deduce if a module is stuck or not, in seconds. +threshold_stucked_module=600 + +[Module_Mixer] +#Define the configuration of the mixer, possible value: 1, 2 or 3 +operation_mode = 3 +#Define the time that a paste will be considerate duplicate. in seconds (1day = 86400) +ttl_duplicate = 86400 +default_unnamed_feed_name = unnamed_feeder + +[Tracker_Term] +max_execution_time = 120 + +[Tracker_Regex] +max_execution_time = 60 + +##### Redis / Valkey ##### +[Redis_Cache] +host = cache +port = 6379 +db = 0 + +[Redis_Log] +host = log +port = 6379 +db = 0 + +[Redis_Log_submit] +host = log-submit +port = 6379 +db = 1 + +[Redis_Queues] +host = queues +port = 6379 +db = 0 + +[Redis_Process] +host = process +port = 6379 +db = 2 + +[Redis_Mixer_Cache] +host = mixer-cache +port = 6379 +db = 1 + +##### KVROCKS ##### + +[Kvrocks_DB] +host = kvrocks +port = 6383 +password = ail + +[Kvrocks_Duplicates] +host = kvrocks +port = 6383 +password = ail_dups + +[Kvrocks_Correlations] +host = kvrocks +port = 6383 +password = ail_correls + +[Kvrocks_Crawler] +host = kvrocks +port = 6383 +password = ail_crawlers + +[Kvrocks_Languages] +host = kvrocks +port = 6383 +password = ail_langs + +[Kvrocks_Objects] +host = kvrocks +port = 6383 +password = ail_objs + +[Kvrocks_Relationships] +host = kvrocks +port = 6383 +password = ail_rels + +[Kvrocks_Searchs] +host = kvrocks +port = 6383 +password = ail_searchs + +[Kvrocks_Timeline] +host = kvrocks +port = 6383 +password = ail_tls + +[Kvrocks_Stats] +host = kvrocks +port = 6383 +password = ail_stats + +[Kvrocks_Tags] +host = kvrocks +port = 6383 +password = ail_tags + +[Kvrocks_Trackers] +host = kvrocks +port = 6383 +password = ail_trackers + +##### - ##### + +[Url] +cc_critical = DE + +[DomClassifier] +#cc = DE +#cc_tld = r'\.de$' +cc = +cc_tld = +dns = 8.8.8.8 + + +[Mail] +dns = 8.8.8.8 + +# Indexer configuration +[Indexer] +meilisearch = False +meilisearch_url = http://localhost:7700 +meilisearch_key = ailmeilisearchpassword + +[ailleakObject] +maxDuplicateToPushToMISP=10 + +############################################################################### + +# For multiple feed, add them with "," without space +# e.g.: tcp://127.0.0.1:5556,tcp://127.0.0.1:5557 +[ZMQ_Global] +# address = tcp://127.0.0.1:5556,tcp://crf.circl.lu:5556 +address = tcp://127.0.0.1:5556 +channel = 102 +bind = tcp://127.0.0.1:5556 + +[RedisPubSub] +host = queues +port = 6379 +db = 0 + +[Crawler] +activate_crawler = False +default_depth_limit = 1 +default_har = True +default_screenshot = True +onion_proxy = onion.foundation +ail_url_to_push_onion_discovery = +ail_key_to_push_onion_discovery = + +[Translation] +libretranslate = + +[IP] +# list of comma-separated CIDR that you wish to be alerted for. e.g: +#networks = 192.168.34.0/24,10.0.0.0/8,192.168.33.0/24 +networks = + +[SubmitPaste] +# 1 Mb Max text paste size for text submission +TEXT_MAX_SIZE = 1000000 +# 1 Gb Max file size for file submission +FILE_MAX_SIZE = 1000000000 +# Managed file extenions for file submission, comma separated +# TODO add zip, gz and tar.gz +FILE_ALLOWED_EXTENSIONS = txt,sh,pdf,html,json diff --git a/other_installers/docker/docker-compose.yml b/other_installers/docker/docker-compose.yml index b10b7afd..8a3174db 100644 --- a/other_installers/docker/docker-compose.yml +++ b/other_installers/docker/docker-compose.yml @@ -1,523 +1,109 @@ -version: '3' +version: '3.8' + +# Define a reusable base for the valkey services to keep the config DRY +x-valkey-service: &valkey-service + image: ${VALKEY_IMAGE:-valkey/valkey:8} + restart: unless-stopped + networks: + - ail-network + healthcheck: + test: ["CMD", "valkey-cli", "ping"] + interval: 5s + timeout: 3s + retries: 5 + services: - ardb: - entrypoint: - - ardb-server - - /opt/AIL/configs/6382.conf + kvrocks: + image: ${KVROCKS_IMAGE:-apache/kvrocks:2.14.0} + container_name: ail-kvrocks + restart: unless-stopped + networks: + - ail-network + volumes: + - kvrocks_data:/var/lib/kvrocks + - ./kvrocks.conf:/var/lib/kvrocks/kvrocks.conf:Z healthcheck: - test: ["CMD", "redis-cli", "-p", "6382", "ping"] - interval: 30s - timeout: 10s + test: ["CMD", "redis-cli", "-h", "localhost", "-p", "6383", "-a", "ail", "ping"] + interval: 5s + timeout: 3s retries: 5 - network_mode: service:flask - image: ail-framework - volumes: - - ./configs:/opt/AIL/configs:ro - crawler: - depends_on: - - redis-log - entrypoint: - - /opt/AIL/bin/Crawler.py - - onion - - "8050" - network_mode: service:flask - image: ail-framework - volumes: - - ./CRAWLED_SCREENSHOT/:/opt/AIL/CRAWLED_SCREENSHOT - - ./PASTES/:/opt/AIL/PASTES - - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro - working_dir: /opt/AIL/bin - flask: - build: . - entrypoint: - - /opt/AIL/var/www/Flask_server.py + + cache: + <<: *valkey-service + container_name: ail-cache + volumes: + - cache_data:/data + + log: + <<: *valkey-service + container_name: ail-log + volumes: + - log_data:/data + + log-submit: + <<: *valkey-service + container_name: ail-log-submit + volumes: + - log-submit_data:/data + + queues: + <<: *valkey-service + container_name: ail-queues + volumes: + - queues_data:/data + + process: + <<: *valkey-service + container_name: ail-process + volumes: + - process_data:/data + + mixer-cache: + <<: *valkey-service + container_name: ail-mixer-cache + volumes: + - mixer-cache_data:/data + + ail: + image: ${AIL_IMAGE:-localhost/ail} + container_name: ail + restart: unless-stopped ports: - "7000:7000" - image: ail-framework - volumes: - - ./CRAWLED_SCREENSHOT/:/opt/AIL/CRAWLED_SCREENSHOT - - ./PASTES/:/opt/AIL/PASTES - - ./indexdir:/opt/AIL/indexdir - - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro - working_dir: /opt/AIL/var/www - log-queue: - depends_on: - - redis-log - entrypoint: - - /opt/AIL/AILENV/bin/log_subscriber - - -p - - "6380" - - -c - - Queing - - -l - - /opt/AIL/logs/ - network_mode: service:flask - image: ail-framework - volumes: - - ./configs:/opt/AIL/configs:ro - log-script: - depends_on: - - redis-log - entrypoint: - - /opt/AIL/AILENV/bin/log_subscriber - - -p - - "6380" - - -c - - Script - - -l - - /opt/AIL/logs/ - network_mode: service:flask - image: ail-framework - volumes: - - ./configs:/opt/AIL/configs:ro - pystemon: - depends_on: - - redis-log - entrypoint: - - /opt/pystemon/pystemon.py - network_mode: service:flask - image: ail-framework - volumes: - - ./PASTES/:/opt/AIL/PASTES - - ./pystemon/archives:/opt/pystemon/archive - - ./pystemon/proxies.txt:/opt/pystemon/proxies.txt:ro - - ./pystemon/pystemon.yaml:/opt/pystemon/pystemon.yaml:ro - working_dir: /opt/pystemon - pystemon-feeder: - depends_on: - - redis-log - entrypoint: - - /opt/AIL/bin/feeder/pystemon-feeder.py - network_mode: service:flask - image: ail-framework - volumes: - - ./PASTES/:/opt/AIL/PASTES - - ./pystemon/archives:/opt/pystemon/archive - - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro - working_dir: /opt/AIL/bin - queues: - depends_on: - - redis-log - entrypoint: - - /opt/AIL/bin/launch_queues.py - network_mode: service:flask - image: ail-framework - volumes: - - ./PASTES/:/opt/AIL/PASTES - - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro - working_dir: /opt/AIL/bin - redis-cache: - entrypoint: - - redis-server - - /opt/AIL/configs/6379.conf - healthcheck: - test: ["CMD", "redis-cli", "-p", "6379", "ping"] - interval: 30s - timeout: 10s - retries: 5 - image: ail-framework - network_mode: service:flask - volumes: - - ./configs:/opt/AIL/configs:ro - redis-log: - entrypoint: - - redis-server - - /opt/AIL/configs/6380.conf - healthcheck: - test: ["CMD", "redis-cli", "-p", "6380", "ping"] - interval: 30s - timeout: 10s - retries: 5 - network_mode: service:flask - image: ail-framework - volumes: - - ./configs:/opt/AIL/configs:ro - redis-mixer-cache: - entrypoint: - - redis-server - - /opt/AIL/configs/6381.conf - healthcheck: - test: ["CMD", "redis-cli", "-p", "6381", "ping"] - interval: 30s - timeout: 10s - retries: 5 - image: ail-framework - network_mode: service:flask - volumes: - - ./configs:/opt/AIL/configs:ro - script-alerthandler: - depends_on: - - redis-log - entrypoint: - - /opt/AIL/bin/alertHandler.py - network_mode: service:flask - image: ail-framework - volumes: - - ./PASTES/:/opt/AIL/PASTES - - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro - working_dir: /opt/AIL/bin - script-apikey: - depends_on: - - redis-log - entrypoint: - - /opt/AIL/bin/ApiKey.py - network_mode: service:flask - image: ail-framework - volumes: - - ./PASTES/:/opt/AIL/PASTES - - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro - working_dir: /opt/AIL/bin - script-bankaccount: - depends_on: - - redis-log - entrypoint: - - /opt/AIL/bin/BankAccount.py - network_mode: service:flask - image: ail-framework - volumes: - - ./PASTES/:/opt/AIL/PASTES - - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro - working_dir: /opt/AIL/bin - script-bitcoin: - depends_on: - - redis-log - entrypoint: - - /opt/AIL/bin/Bitcoin.py - network_mode: service:flask - image: ail-framework - volumes: - - ./PASTES/:/opt/AIL/PASTES - - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro - working_dir: /opt/AIL/bin - script-categ: - depends_on: - - redis-log - entrypoint: - - /opt/AIL/bin/Categ.py - network_mode: service:flask - image: ail-framework - volumes: - - ./PASTES/:/opt/AIL/PASTES - - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro - working_dir: /opt/AIL/bin - script-credential: - depends_on: - - redis-log - entrypoint: - - /opt/AIL/bin/Credential.py - network_mode: service:flask - image: ail-framework - volumes: - - ./PASTES/:/opt/AIL/PASTES - - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro - working_dir: /opt/AIL/bin - script-creditcards: - depends_on: - - redis-log - entrypoint: - - /opt/AIL/bin/CreditCards.py - network_mode: service:flask - image: ail-framework - volumes: - - ./PASTES/:/opt/AIL/PASTES - - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro - working_dir: /opt/AIL/bin - script-curve: - depends_on: - - redis-log - entrypoint: - - /opt/AIL/bin/Curve.py - network_mode: service:flask - image: ail-framework - volumes: - - ./PASTES/:/opt/AIL/PASTES - - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro - working_dir: /opt/AIL/bin - script-curvemanagetopsets: - depends_on: - - redis-log - entrypoint: - - /opt/AIL/bin/CurveManageTopSets.py - network_mode: service:flask - image: ail-framework - volumes: - - ./PASTES/:/opt/AIL/PASTES - - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro - working_dir: /opt/AIL/bin - script-cve: - depends_on: - - redis-log - entrypoint: - - /opt/AIL/bin/CveModule.py - network_mode: service:flask - image: ail-framework - volumes: - - ./PASTES/:/opt/AIL/PASTES - - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro - working_dir: /opt/AIL/bin - script-decoder: - depends_on: - - redis-log - entrypoint: - - /opt/AIL/bin/Decoder.py - network_mode: service:flask - image: ail-framework - volumes: - - ./HASHS:/opt/AIL/HASHS - - ./PASTES/:/opt/AIL/PASTES - - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro - working_dir: /opt/AIL/bin - script-domclassifier: - depends_on: - - redis-log - entrypoint: - - /opt/AIL/bin/DomClassifier.py - network_mode: service:flask - image: ail-framework - volumes: - - ./PASTES/:/opt/AIL/PASTES - - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro - working_dir: /opt/AIL/bin - script-duplicates: - depends_on: - - redis-log - entrypoint: - - /opt/AIL/bin/Duplicates.py - network_mode: service:flask - image: ail-framework - volumes: - - ./PASTES/:/opt/AIL/PASTES - - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro - working_dir: /opt/AIL/bin - script-global: - depends_on: - - redis-log - entrypoint: - - /opt/AIL/bin/Global.py - network_mode: service:flask - image: ail-framework - volumes: - - ./PASTES/:/opt/AIL/PASTES - - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro - working_dir: /opt/AIL/bin - script-indexer: - depends_on: - - redis-log - entrypoint: - - /opt/AIL/bin/Indexer.py - network_mode: service:flask - image: ail-framework - volumes: - - ./PASTES/:/opt/AIL/PASTES - - ./indexdir:/opt/AIL/indexdir - - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro - working_dir: /opt/AIL/bin - script-keys: - depends_on: - - redis-log - entrypoint: - - /opt/AIL/bin/Keys.py - network_mode: service:flask - image: ail-framework - volumes: - - ./PASTES/:/opt/AIL/PASTES - - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro - working_dir: /opt/AIL/bin - script-libinjection: - depends_on: - - redis-log - entrypoint: - - /opt/AIL/bin/LibInjection.py - network_mode: service:flask - image: ail-framework - volumes: - - ./PASTES/:/opt/AIL/PASTES - - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro - working_dir: /opt/AIL/bin - script-lines: - depends_on: - - redis-log - entrypoint: - - /opt/AIL/bin/Lines.py - network_mode: service:flask - image: ail-framework - volumes: - - ./PASTES/:/opt/AIL/PASTES - - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro - working_dir: /opt/AIL/bin - script-mail: - depends_on: - - redis-log - entrypoint: - - /opt/AIL/bin/Mail.py - network_mode: service:flask - image: ail-framework - volumes: - - ./PASTES/:/opt/AIL/PASTES - - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro - working_dir: /opt/AIL/bin - script-mispthehivefeeder: - depends_on: - - redis-log - entrypoint: - - /opt/AIL/bin/MISP_The_Hive_feeder.py - network_mode: service:flask - image: ail-framework - volumes: - - ./PASTES/:/opt/AIL/PASTES - - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro - working_dir: /opt/AIL/bin - script-mixer: - depends_on: - - redis-mixer-cache - entrypoint: - - /opt/AIL/bin/Mixer.py - network_mode: service:flask - image: ail-framework - volumes: - - ./PASTES/:/opt/AIL/PASTES - - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro - working_dir: /opt/AIL/bin - script-modulestats: - depends_on: - - redis-log - entrypoint: - - /opt/AIL/bin/ModuleStats.py - network_mode: service:flask - image: ail-framework - volumes: - - ./PASTES/:/opt/AIL/PASTES - - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro - working_dir: /opt/AIL/bin - script-onion: - depends_on: - - redis-log - entrypoint: - - /opt/AIL/bin/Onion.py - network_mode: service:flask - image: ail-framework - volumes: - - ./PASTES/:/opt/AIL/PASTES - - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro - working_dir: /opt/AIL/bin - script-phone: - depends_on: - - redis-log - entrypoint: - - /opt/AIL/bin/Phone.py - network_mode: service:flask - image: ail-framework - volumes: - - ./PASTES/:/opt/AIL/PASTES - - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro - working_dir: /opt/AIL/bin - script-regexfortermsfrequency: - depends_on: - - redis-log - entrypoint: - - /opt/AIL/bin/RegexForTermsFrequency.py - network_mode: service:flask - image: ail-framework - volumes: - - ./PASTES/:/opt/AIL/PASTES - - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro - working_dir: /opt/AIL/bin - script-release: - depends_on: - - redis-log - entrypoint: - - /opt/AIL/bin/Release.py - network_mode: service:flask - image: ail-framework - volumes: - - ./PASTES/:/opt/AIL/PASTES - - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro - working_dir: /opt/AIL/bin - script-sentimentanalysis: - depends_on: - - redis-log - entrypoint: - - /opt/AIL/bin/SentimentAnalysis.py - network_mode: service:flask - image: ail-framework - volumes: - - ./PASTES/:/opt/AIL/PASTES - - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro - working_dir: /opt/AIL/bin - script-setfortermsfrequency: - depends_on: - - redis-log - entrypoint: - - /opt/AIL/bin/SetForTermsFrequency.py - network_mode: service:flask - image: ail-framework - volumes: - - ./PASTES/:/opt/AIL/PASTES - - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro - working_dir: /opt/AIL/bin - script-sqlinjectiondetection: - depends_on: - - redis-log - entrypoint: - - /opt/AIL/bin/SQLInjectionDetection.py - network_mode: service:flask - image: ail-framework - volumes: - - ./PASTES/:/opt/AIL/PASTES - - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro - working_dir: /opt/AIL/bin - script-submitpaste: - depends_on: - - redis-log - entrypoint: - - /opt/AIL/bin/submit_paste.py - network_mode: service:flask - image: ail-framework - volumes: - - ./PASTES/:/opt/AIL/PASTES - - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro - working_dir: /opt/AIL/bin - script-tags: - depends_on: - - redis-log - entrypoint: - - /opt/AIL/bin/Tags.py - network_mode: service:flask - image: ail-framework - volumes: - - ./PASTES/:/opt/AIL/PASTES - - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro - working_dir: /opt/AIL/bin - script-tokenize: - depends_on: - - redis-log - entrypoint: - - /opt/AIL/bin/Tokenize.py - network_mode: service:flask - image: ail-framework - volumes: - - ./PASTES/:/opt/AIL/PASTES - - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro - working_dir: /opt/AIL/bin - script-web: - depends_on: - - redis-log - entrypoint: - - /opt/AIL/bin/Web.py - network_mode: service:flask - image: ail-framework - volumes: - - ./PASTES/:/opt/AIL/PASTES - - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro - working_dir: /opt/AIL/bin - script-webstats: - depends_on: - - redis-log - entrypoint: - - /opt/AIL/bin/WebStats.py - network_mode: service:flask - image: ail-framework - volumes: - - ./PASTES/:/opt/AIL/PASTES - - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro - working_dir: /opt/AIL/bin + networks: + - ail-network + environment: + - SKIP_LAUNCH_REDIS=true + - SKIP_LAUNCH_KVROCKS=true + - SKIP_CHECK_REDIS=true + - SKIP_CHECK_KVROCKS=true + volumes: + - ./core.cfg:/home/ail/ail-framework/configs/core.cfg:Z + depends_on: + kvrocks: + condition: service_healthy + cache: + condition: service_healthy + log: + condition: service_healthy + log-submit: + condition: service_healthy + queues: + condition: service_healthy + process: + condition: service_healthy + mixer-cache: + condition: service_healthy + +networks: + ail-network: + driver: bridge + +volumes: + kvrocks_data: + cache_data: + log_data: + log-submit_data: + queues_data: + process_data: + mixer-cache_data: \ No newline at end of file diff --git a/other_installers/docker/kvrocks.conf b/other_installers/docker/kvrocks.conf new file mode 100644 index 00000000..971c56fb --- /dev/null +++ b/other_installers/docker/kvrocks.conf @@ -0,0 +1,185 @@ +################################ GENERAL ##################################### + +# BINDING: Set to 0.0.0.0 to allow access from outside the Docker container. +bind 0.0.0.0 + +# PORT: Set to 6383 to match your original AIL configuration. +port 6383 + +# TIMEOUT: Close the connection after a client is idle for N seconds (0 to disable) +timeout 0 + +# WORKERS: The number of worker threads. +workers 8 + +# DAEMONIZE: Always 'no' for Docker. +daemonize no + +# SOCKET: Support for passing file descriptors (Standard in newer versions) +socket-fd -1 + +################################ AUTHENTICATION ############################## + +# PASSWORD: Set to 'ail' to match your original configuration. +requirepass ail + +# MASTERAUTH: If you use replication, set this to the master's password. +# masterauth ail + +################################ CLUSTER ##################################### + +cluster-enabled no +persist-cluster-nodes-enabled yes + +################################ REPLICATION ################################# + +# Namespace replication settings +repl-namespace-enabled no + +# Standard limits +maxclients 10000 +db-name change.me.db + +# DIRECTORY: Standard path for Docker persistence. +# Ensure your docker-compose maps a volume to this path. +dir /var/lib/kvrocks + +# LOGGING: Standard Docker logging (stdout) +log-dir stdout +log-level info +log-retention-days -1 + +# SLAVE SETTINGS +slave-read-only yes +slave-priority 100 +slave-serve-stale-data yes +slave-empty-db-before-fullsync no +purge-backup-on-fullsync no + +# REPLICATION TIMEOUTS (Newer defaults) +replication-connect-timeout-ms 3100 +replication-recv-timeout-ms 3200 +replication-group-sync no +replication-no-slowdown yes + +################################ LIMITS & PROTOCOLS ########################## + +# Max replication speed (0 = unlimited) +max-replication-mb 0 +max-io-mb 0 +max-db-size 0 + +# Backup retention +max-backup-to-keep 1 +max-backup-keep-hours 24 + +# Compatibility settings +redis-cursor-compatible yes +resp3-enabled yes +json-max-nesting-depth 1024 +json-storage-format json + +# Experimental Transactional Context (Default: no) +txn-context-enabled no +lua-strict-key-accessing no + +################################ SECURITY (TLS) ############################## + +# TLS is disabled by default (port 0) +tls-port 0 +tls-auth-clients no +tls-prefer-server-ciphers yes +tls-session-caching no +tls-replication no + +################################ LOGGING & MONITORING ######################## + +# SLOW LOG +slowlog-log-slower-than 100000 +slowlog-max-len 128 +slowlog-dump-logfile-level off + +# PERF LOG +profiling-sample-ratio 0 +profiling-sample-record-max-len 256 +profiling-sample-record-threshold-ms 100 + +################################ CRON (MAINTENANCE) ########################## + +# Compaction Checker (Newer Syntax) +# Checks for compaction needs between 0:00 and 7:00 AM daily +compaction-checker-cron * 0-7 * * * + +# Auto-resize estimated keyspace +# dbsize-scan-cron 0 * * * * + +################################ MIGRATION ################################### + +migrate-type raw-key-value +migrate-speed 4096 +migrate-pipeline-size 16 +migrate-sequence-gap 10000 +migrate-batch-size-kb 16 +migrate-batch-rate-limit-mb 16 + +################################ ROCKSDB TUNING ############################## +# Modern defaults for performance + +rocksdb.block_cache_size 4096 +rocksdb.block_cache_type lru +rocksdb.max_open_files 8096 +rocksdb.write_buffer_size 64 +rocksdb.target_file_size_base 128 +rocksdb.max_write_buffer_number 4 +rocksdb.min_write_buffer_number_to_merge 1 +rocksdb.max_background_jobs 4 +rocksdb.max_subcompactions 2 +rocksdb.wal_compression no +rocksdb.max_total_wal_size 512 +rocksdb.dump_malloc_stats yes +rocksdb.wal_ttl_seconds 10800 +rocksdb.wal_size_limit_mb 16384 +rocksdb.block_size 16384 +rocksdb.cache_index_and_filter_blocks yes +rocksdb.compression snappy +rocksdb.compression_level 32767 +rocksdb.compaction_readahead_size 2097152 +rocksdb.compression_start_level 2 + +# ASYNC IO: Enabled for performance (Requires modern Kernel/Docker) +rocksdb.read_options.async_io yes + +rocksdb.write_options.sync no +rocksdb.write_options.disable_wal no +rocksdb.write_options.no_slowdown no +rocksdb.rate_limiter_auto_tuned yes +rocksdb.partition_filters yes + +# BLOB DB (Large value optimization) +rocksdb.enable_blob_files no +rocksdb.min_blob_size 4096 +rocksdb.blob_file_size 268435456 +rocksdb.enable_blob_garbage_collection yes +rocksdb.blob_garbage_collection_age_cutoff 25 +enable-blob-cache no + +# LEVEL COMPACTION +rocksdb.level_compaction_dynamic_level_bytes yes +rocksdb.max_bytes_for_level_base 268435456 +rocksdb.max_bytes_for_level_multiplier 10 + +################################ NAMESPACES ################################## +# Imported from your AIL configuration + +namespace.cor ail_correls +namespace.crawl ail_crawlers +namespace.db ail_datas +namespace.dup ail_dups +namespace.lg ail_langs +namespace.obj ail_objs +namespace.rel ail_rels +namespace.se ail_searchs +namespace.stat ail_stats +namespace.tag ail_tags +namespace.tl ail_tls +namespace.track ail_trackers \ No newline at end of file diff --git a/other_installers/docker/update.cfg b/other_installers/docker/update.cfg new file mode 100644 index 00000000..0fea91c3 --- /dev/null +++ b/other_installers/docker/update.cfg @@ -0,0 +1,4 @@ +[Update] +auto_update = False +upstream = upstream +update-fork = False