diff --git a/mint.json b/mint.json
index 02abe70..2b433b5 100644
--- a/mint.json
+++ b/mint.json
@@ -53,21 +53,44 @@
"kubernetes"
]
},
- "benchmarks"
+ "benchmarks",
+ {
+ "group": "Troubleshooting",
+ "icon": "hammer",
+ "pages": [
+ "troubleshooting-guide"
+ ]
+ }
]
},
{
"group": "Tutorials",
"pages": [
{
- "group": "Using the dashboard",
+ "group": "Use the dashboard",
"icon": "chart-column",
"pages": [
"request-graph",
"adding-filter-clauses"
]
+ },
+ {
+ "group": "Filter traffic to Subtrace",
+ "icon": "filter",
+ "pages": [
+ "proxy-rules-guide"
+ ]
}
]
+ },
+ {
+ "group": "References",
+ "pages": [
+ "references/subtrace-run",
+ "references/subtrace-proxy",
+ "references/subtrace-worker",
+ "references/subtrace-version"
+ ]
}
],
"footerSocials": {
diff --git a/proxy-rules-guide.mdx b/proxy-rules-guide.mdx
new file mode 100644
index 0000000..b1fe95b
--- /dev/null
+++ b/proxy-rules-guide.mdx
@@ -0,0 +1,165 @@
+---
+title: "Writing Rules for Subtrace Proxy"
+description: "Learn how to use the subtrace proxy command to control which HTTP traffic is traced"
+lastUpdated: "2024-12-17"
+---
+# Writing Rules for Subtrace Proxy: A Beginner's Guide
+
+## Introduction
+
+When running subtrace proxy to monitor your HTTP traffic, you may not want to trace every single request - some might be too noisy (like health checks), while others may contain sensitive data. Subtrace lets you control this using JavaScript rules.
+
+## Basic Setup
+
+First, let's set up a basic proxy that forwards traffic from port 8080 to our API running on port 3000:
+
+```bash
+# Create a rules file
+touch rules.js
+
+# Start the proxy
+subtrace proxy \
+ -listen localhost:8080 \
+ -remote localhost:3000 \
+ -config rules.js
+```
+
+## Writing Your First Rule
+
+Rules are written using the `trace()` function. Let's start with a simple rule that traces all requests:
+
+```javascript
+// rules.js
+trace("*", "*", function() {
+ return true;
+});
+```
+
+The `trace()` function takes three arguments:
+1. HTTP method pattern (like "GET", "POST", "*" for all)
+2. Path pattern (like "/api/users", "/health", "*" for all)
+3. A function that returns true to trace the request, false to ignore it
+
+## Building More Specific Rules
+
+Let's build rules for our ecommerce API. We want to:
+- Trace all product-related requests
+- Skip health checks
+- Sample only 10% of static asset requests
+- Always trace error responses
+
+```javascript
+// rules.js
+
+// Trace all product-related endpoints
+trace("*", "/api/products/*", function() {
+ return true;
+});
+
+// Skip health checks completely
+trace("GET", "/health", function() {
+ return false;
+});
+
+// Sample 10% of static asset requests
+trace("GET", "/static/*", {
+ sample: 0.1
+});
+
+// Trace any request that returns an error
+trace("*", "*", function(req, resp) {
+ return resp.statusCode >= 400;
+});
+```
+
+## Pattern Matching
+
+The method and path patterns support:
+- Exact matches: "GET", "/api/users"
+- Wildcards: "*" matches anything
+- Suffixes: "/api/*" matches any path starting with /api/
+
+## Adding Conditions
+
+You can make more complex decisions in your rule functions. Let's expand our example:
+
+```javascript
+// rules.js
+
+// Trace slow requests that take > 500ms
+trace("*", "*", function(req, resp) {
+ return req.duration > 500;
+});
+
+// Trace requests with specific header
+trace("POST", "/api/*", function(req) {
+ return req.headers["x-trace-this"] === "true";
+});
+
+// Combine multiple conditions
+trace("PUT", "/api/orders/*", function(req, resp) {
+ return resp.statusCode >= 400 || // Error responses
+ req.duration > 1000 || // Slow requests
+ req.body.priority === "high"; // High priority orders
+});
+```
+
+## Rule Evaluation Order
+
+Rules are evaluated in the order they appear in your file. Once a rule matches and returns true, that decision is final. Use this to create precedence:
+
+```javascript
+// rules.js
+
+// First, skip health checks unconditionally
+trace("GET", "/health", () => false);
+
+// Then apply sampling to everything else
+trace("*", "*", { sample: 0.1 });
+```
+
+## Reference
+
+Common patterns:
+
+```javascript
+// Match specific HTTP methods
+trace("GET", "...")
+trace("POST", "...")
+trace("PUT", "...")
+trace("DELETE", "...")
+
+// Match paths
+trace("*", "/exact/path")
+trace("*", "/api/*") // Everything under /api
+trace("*", "*.jpg") // All jpg files
+trace("*", "/users/*/posts") // Match /users/123/posts
+
+// Sampling
+trace("*", "*", { sample: 0.5 }) // 50% of requests
+
+// Access request/response info
+trace("*", "*", function(req, resp) {
+ req.method // HTTP method
+ req.path // Request path
+ req.headers // Request headers
+ req.body // Request body
+ req.duration // Request duration in ms
+
+ resp.statusCode // Response status code
+ resp.headers // Response headers
+ resp.body // Response body
+})
+```
+
+## Best Practices
+
+1. Start with broad rules and refine them
+2. Use sampling for high-volume endpoints
+3. Always skip health check endpoints
+4. Put most specific rules first
+5. Use meaningful patterns that map to your API structure
+6. Don't overload your rules with complex logic
+7. Monitor the impact of your rules on system performance
+
+The key is to find the right balance between visibility and performance for your specific use case. Start simple and iterate based on your needs.
diff --git a/quickstart.mdx b/quickstart.mdx
index 78f2a4e..7e1936f 100644
--- a/quickstart.mdx
+++ b/quickstart.mdx
@@ -1,37 +1,132 @@
---
title: "Subtrace Quickstart"
sidebarTitle: "Quickstart"
-description: "Welcome! Follow the steps below to get started with Subtrace."
+description: "Welcome! Follow the steps below to get started tracing your HTTP/HTTPS requests and service calls."
icon: "rocket"
---
**Step 1**: Go to [https://subtrace.dev/login](https://subtrace.dev/login) and log in with your single sign-on (SSO) identity provider.
-**Step 2**: On the **Let's set up your database** page, follow the onboarding instructions to start the database worker service where your backend requests will be stored.
+**Step 2**: Visit the [Let's set up your database](https://subtrace.dev/onboarding) onboarding page and start the database worker service where your backend requests will be stored.
+
-**Step 3**: Go to the **Tokens** page on the dashboard and click **Create tracer token**.
+
+ ```bash
+ docker run -d \
+ -e SUBTRACE_TOKEN=subt_REDACTED_WORKER_TOKEN \
+ -v subtrace:/var/lib/clickhouse \
+ subtrace.dev/worker:latest
+ ```
+
+ Run this command with the **worker token** from the [Tokens](https://subtrace.dev/dashboard/workspace/tokens) page.
+
+
+ This is required to authenticate your cloud and self-hosted deployments of Subtrace.
+
+
+
+
+**Step 3**: Go to the [Tokens](https://subtrace.dev/dashboard/workspace/tokens) page on the dashboard and click **Create tracer token**.
-**Step 4**: Install the latest version of Subtrace, set the `SUBTRACE_TOKEN` environment variable to the token you generated in the previous step, and use `subtrace run` to start your app.
+**Step 4**: Install the latest version of Subtrace, set the `SUBTRACE_TOKEN` environment variable to the **tracer token** you generated in step 3, and use `subtrace run` to start your app.
+
+ ```bash bash
+ # replace amd64 with arm64 if applicable
+ curl -fsSLO https://subtrace.dev/download/latest/linux/amd64/subtrace && chmod +x ./subtrace
+
+ # this is just an example, use the token you generated in Step 3
+ export SUBTRACE_TOKEN=subt_REDACTED_TRACER_TOKEN
+
+ # start your app like you normally do, but using Subtrace
+ ./subtrace run -- node ./app.js
+ ```
+ ```javascript app.js
+ try {
+ // Successful request
+ console.log("Making successful request...");
+ const successResponse = await fetch('http://jsonplaceholder.typicode.com/posts/1');
+ console.log(`Success! Status code: ${successResponse.status}`);
+ const successData = await successResponse.json();
+ console.log(`Response:`, successData);
+ console.log('');
+
+ // Failed request (intentionally wrong URL)
+ console.log("Making request that will fail...");
+ try {
+ const failedResponse = await fetch('http://jsonplaceholder.typicode.com/nonexistent', { method: 'POST' });
+ console.log(`Status code: ${failedResponse.status}`);
+ } catch (error) {
+ console.log('Request failed:', error.message);
+ }
+ } catch (error) {
+ console.error('Unexpected error:', error.message);
+ }
+ ```
+
+ ```python main.py
+ import requests
+
+ # Successful request
+ print("Making successful request...")
+ response = requests.get('https://jsonplaceholder.typicode.com/posts/1')
+ print(f"Success! Status code: {response.status_code}")
+ print(f"Response: {response.json()}\n")
+
+ # Failed request (intentionally wrong URL)
+ print("Making request that will fail...")
+ try:
+ response = requests.post('https://jsonplaceholder.typicode.com/nonexistent')
+ print(f"Status code: {response.status_code}")
+ except requests.exceptions.RequestException as e:
+ print(f"Request failed as expected: {e}")
+ ```
+
+ The double dash (`--`) is required to separate subtrace flags from the command you want to run (read more in the [subtrace run reference](/references/subtrace-run-doc#usage)).
+
+**Step 5** *(optional)*: Add the Subtrace executable to your `$PATH` so you can run `subtrace` from anywhere.
+ - Validate that `/usr/local/bin` exists in your `$PATH`.
+
+ ```bash bash
+ echo $PATH
+ ```
+ ```bash output
+ /usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin
+ ```
+
-```bash
-# replace amd64 with arm64 if applicable
-curl -fsSLO https://subtrace.dev/download/latest/linux/amd64/subtrace && chmod +x ./subtrace
+ - Move the `subtrace` executable to `/usr/local/bin`.
-# this is just an example, use the token you generated in Step 3
-export SUBTRACE_TOKEN=subt_CbBZqZousFZ2IBIATEgKCNPvAtZuaRxbtdlzlzmUqNg
+ ```bash
+ sudo mv ./subtrace /usr/local/bin/subtrace
+ ```
+ - Test that subtrace still works.
-# start your app like you normally do, but using Subtrace
-./subtrace run -- node ./app.js
-```
+
+ ```bash bash
+ subtrace version
+ ```
+ ```bash output
+ b149
+ commit 989a856 at 2024-12-13T19:19:15Z
+ built with go1.23.4 linux/amd64 at 2024-12-13T19:19:30Z hash 63ec488edc27699bf65d31a26698d5d7b175afa39ac91554b81acc83d01c999e
+ kernel Linux 6.11.0-13-generic on x86_64
+ running on linux/amd64 with uid 1000 gid 1000
+ effective caps 0x0000000000000000 -chown -dac_override -dac_read_search -fowner -fsetid -kill -setgid -setuid -setpcap -linux_immutable -net_bind_service -net_broadcast -net_admin -net_raw -ipc_lock -ipc_owner -sys_module -sys_rawio -sys_chroot -sys_ptrace -sys_pacct -sys_admin -sys_boot -sys_nice -sys_resource -sys_time -sys_tty_config -mknod -lease -audit_write -audit_control -setfcap -mac_override -mac_admin -syslog -wake_alarm -block_suspend -audit_read -perfmon -bpf -checkpoint_restore
+ ```
+
Congratulations! Your app is now running under Subtrace. Your requests should automatically appear in the [Subtrace dashboard](https://subtrace.dev/dashboard/).
While this is sufficient to get started with Subtrace, you should install Subtrace everywhere in your infrastructure to get the most out of it. Pick the one that matches your situation best to learn more:
-- [Installing on Kubernetes](/kubernetes)
+
+
+ Install Subtrace on Kubernetes to observe all your service calls.
+
+
If your infra uses something that's not listed, we'd love to understand your usecase and help you get started with Subtrace. Join our [discord server](https://subtrace.dev/discord) or email us at [support@subtrace.dev](support@subtrace.dev) to get in touch.
diff --git a/references/subtrace-proxy.mdx b/references/subtrace-proxy.mdx
new file mode 100644
index 0000000..b106bca
--- /dev/null
+++ b/references/subtrace-proxy.mdx
@@ -0,0 +1,161 @@
+---
+title: 'subtrace proxy'
+description: 'Learn how to use the subtrace proxy command to trace HTTP traffic through a reverse proxy'
+---
+
+The `subtrace proxy` command creates a reverse proxy that traces HTTP traffic between clients and a remote server. It can filter and analyze requests based on configurable rules.
+
+## Command Structure
+
+```bash
+subtrace proxy [flags]
+```
+
+## Flags
+
+| Flag | Description | Default |
+|------|-------------|---------|
+| `-config` | Path to configuration file for proxy rules | `""` |
+| `-listen` | Local address to listen on (required) | `""` |
+| `-remote` | Remote address to forward requests to (required) | `""` |
+| `-log` | Log trace events to stderr | `false` if `SUBTRACE_TOKEN` is set, `true` otherwise |
+| `-v` | Enable verbose debug logging | `false` |
+
+## Examples
+
+### Basic Proxy
+
+Forward local traffic to a remote API:
+```bash
+subtrace proxy -listen localhost:8080 -remote api.example.com
+```
+
+### With Configuration File
+
+Use rules to filter requests:
+```bash
+subtrace proxy -listen :8080 -remote api.internal -config proxy-rules.js
+```
+
+### Development Setup
+
+Log locally without sending to Subtrace:
+```bash
+subtrace proxy -listen localhost:3000 -remote localhost:8080 -log
+```
+
+### Debug Mode
+
+Enable verbose logging:
+```bash
+subtrace proxy -v -listen :8080 -remote api.example.com
+```
+
+## Writing Rules in a Configuration File
+
+Refer to the [Proxy Rules Guide](/proxy-rules-guide) guide for more information on how to write rules.
+
+## Environment Variables
+
+| Variable | Description |
+|----------|-------------|
+| `SUBTRACE_TOKEN` | Authentication token for Subtrace API |
+| `SUBTRACE_ENDPOINT` | Custom API endpoint (defaults to https://subtrace.dev) |
+| `SUBTRACE_PROXY_LOG` | Alternative to -log flag |
+
+## Captured Data
+
+The proxy captures:
+
+- Full request and response headers
+- Request and response bodies
+- Timing information
+- TLS details (for HTTPS)
+- Source and destination addresses
+- HTTP protocol details
+
+## Features
+
+- HTTP/1.x and HTTP/2 support
+- TLS/HTTPS proxying
+- Request filtering and sampling
+- HAR (HTTP Archive) format capture
+- Automatic header sanitization
+- Configurable logging
+
+## Common Error Messages
+
+```bash
+# Missing required flags
+error: missing -listen and -remote
+
+# Configuration file error
+error: parse config: invalid syntax in rules.js
+
+# Network error
+error: listen tcp :8080: bind: address already in use
+```
+
+## Best Practices
+
+1. Use specific listen addresses for security
+2. Configure sampling rates for high-traffic services
+3. Filter out sensitive endpoints
+4. Monitor proxy resource usage
+5. Use verbose logging for debugging
+6. Regularly rotate/flush trace data
+7. Set appropriate timeouts
+
+## Limitations
+
+- Single proxy instance per listen address
+- Memory usage scales with concurrent connections
+- TLS interception requires additional setup
+- Rule evaluation adds slight latency
+
+## Security Considerations
+
+- Proxy can see all HTTP traffic (unless you use rules to filter)
+- TLS interception requires trust configuration
+- Consider network access controls
+- Protect configuration files
+- Sanitize sensitive headers
+
+## Example Configuration Patterns
+
+Refer to the [Proxy Rules Guide](/proxy-rules-guide) guide for more example configurations on how to write rules.
+
+### Basic Load Balancer
+
+```javascript
+trace("*", "*", {
+ sample: 0.01 // Sample 1% of all traffic
+})
+```
+
+### API Monitoring
+
+```javascript
+trace("POST", "/api/v1/*", true) // Trace all API calls
+trace("GET", "/static/*", false) // Ignore static content
+```
+
+### Error Tracking
+
+```javascript
+trace("*", "*", function(req, resp) {
+ return resp.statusCode >= 400 // Trace errors
+})
+```
+
+## Related Commands
+
+- [`subtrace run`](/references/subtrace-run) - Run a process with tracing enabled
+- [`subtrace worker`](/references/subtrace-worker) - Start a worker node (for self-hosted setups)
+- [`subtrace version`](/references/subtrace-version) - Show version information
+
+## Additional Resources
+
+- [Subtrace Documentation](https://docs.subtrace.dev)
+- [Proxy Rule Writing Guide](/proxy-rules-guide)
+{/* - [Security Best Practices](https://docs.subtrace.dev/security) */}
diff --git a/references/subtrace-run.mdx b/references/subtrace-run.mdx
new file mode 100644
index 0000000..fe40081
--- /dev/null
+++ b/references/subtrace-run.mdx
@@ -0,0 +1,141 @@
+---
+title: 'subtrace run'
+description: 'Run a command with HTTP traffic tracing enabled'
+---
+
+The `subtrace run` command allows you to trace HTTP traffic from any application without modifying its code. It works by intercepting network syscalls at runtime and capturing details about HTTP requests and responses.
+
+## Usage
+
+```bash
+subtrace run [flags] -- [arguments]
+```
+
+
+ The `--` is a [bash builtin command](https://www.gnu.org/software/bash/manual/html_node/Bash-Builtins.html) that is used to indicate that all arguments after it are treated as a command to run, not as arguments for `subtrace run`.
+
+
+## Flags
+
+| Flag | Description | Default |
+|------|-------------|---------|
+| `-log` | Log trace events to stderr | `false` if `SUBTRACE_TOKEN` is set, `true` otherwise |
+| `-payload-limit` | Maximum size in bytes for request/response body capture | 4096 |
+| `-tls` | Enable TLS traffic interception | `true` |
+| `-v` | Enable verbose debug logging | `false` |
+
+## Examples
+
+### Trace a Python HTTP Server
+
+```bash
+subtrace run -- python -m http.server
+```
+
+### Trace cURL Requests
+
+```bash
+subtrace run -- curl https://api.example.com
+```
+
+### Trace a Node.js Application
+
+```bash
+subtrace run -- node server.js
+```
+
+### Trace with Limited Payload Size
+
+```bash
+subtrace run -payload-limit 1024 -- ./myapp
+```
+
+### Trace without TLS Interception
+
+```bash
+subtrace run -tls=false -- ./myapp
+```
+
+## How It Works
+
+When you run an application with `subtrace run`:
+
+1. Subtrace creates a secure sandbox environment using Linux seccomp filters
+2. It intercepts network-related system calls like `socket()`, `connect()`, and `accept()`
+3. For HTTP traffic, it captures:
+ - Request method, path, headers
+ - Response status code, headers
+ - Request and response bodies (up to payload limit)
+ - Timing information
+ - TLS server names and certificates
+4. All captured data is sent to the Subtrace backend for analysis
+
+## Requirements
+
+- Linux kernel version 5.14 or newer
+- `CAP_SYS_PTRACE` capability when running in production
+- Root access or `CAP_SYS_ADMIN` capability for TLS interception
+
+## Environment Variables
+
+| Variable | Description |
+|----------|-------------|
+| `SUBTRACE_TOKEN` | Authentication token for the Subtrace API |
+| `SUBTRACE_ENDPOINT` | Custom API endpoint (defaults to https://subtrace.dev) |
+
+## Limitations
+
+- Works only on Linux systems
+- Cannot trace statically linked binaries
+- May require configuration changes for applications that perform certificate pinning
+
+## Error Handling
+
+If subtrace encounters an error while tracing, it will:
+
+1. Log the error to stderr with context
+2. Continue tracing other requests if possible
+3. Preserve the original application behavior
+
+Common error scenarios:
+
+```bash
+# Missing SUBTRACE_TOKEN
+subtrace: error: SUBTRACE_TOKEN is empty
+
+# Insufficient kernel version
+subtrace: error: unsupported Linux kernel version (got 4.19, want 5.14+)
+
+# Missing capabilities
+subtrace: error: was subtrace started with the SYS_PTRACE capability?
+```
+
+## Example Output
+
+When `-log` is enabled, subtrace prints trace events in a key-value format:
+
+```
+time="2024-03-14T15:04:05Z" event_id="550e8400-e29b-41d4-a716-446655440000" http_req_method="GET" http_req_path="/api/users" http_resp_status_code=200 http_duration=127
+```
+
+## Best Practices
+
+1. Always use the latest version of subtrace for best compatibility
+2. Set appropriate payload size limits based on your application
+3. Consider disabling TLS interception if not needed
+4. Use verbose logging (-v) when debugging trace issues
+5. Set up proper access controls for the Subtrace API token
+
+## Security Considerations
+
+- Subtrace can see all HTTP traffic, including sensitive headers and payloads
+- TLS interception requires trust in Subtrace's certificate authority
+- API tokens should be treated as sensitive credentials
+- Consider using separate tokens for development and production
+
+## Related Commands
+
+- [`subtrace proxy`](/references/subtrace-proxy) - Run a reverse proxy with tracing
+- [`subtrace version`](/references/subtrace-version) - Show version information
+- [`subtrace worker`](/references/subtrace-worker) - Start a worker node (for self-hosted setups)
+
diff --git a/references/subtrace-version.mdx b/references/subtrace-version.mdx
new file mode 100644
index 0000000..958378b
--- /dev/null
+++ b/references/subtrace-version.mdx
@@ -0,0 +1,90 @@
+---
+title: 'subtrace version'
+description: 'Display version and system information about the Subtrace installation'
+---
+
+## Synopsis
+
+```bash
+subtrace version [flags]
+```
+
+## Description
+
+The `subtrace version` command provides detailed information about your Subtrace installation, including build details, system information, and capabilities. This information is particularly useful for troubleshooting, bug reports, and verifying your installation.
+
+## Options
+
+| Flag | Description | Default |
+|--------|---------------------------------|---------|
+| -json | Output version info as JSON | false |
+
+## Output Information
+
+The command outputs the following information:
+- Release version
+- Commit hash and timestamp
+- Build information (Go version, OS, architecture)
+- Executable hash
+- Kernel information
+- Runtime details (UID, GID)
+- Effective capabilities
+
+## Examples
+
+### Basic Usage
+Display version information in standard format:
+```bash
+$ subtrace version
+b123
+ commit abc123def at 2024-03-15T10:30:00Z
+ built with go1.22.0 linux/amd64 at 2024-03-15T10:30:00Z hash 8a7b6c5d4e3f...
+ kernel Linux 5.15.0 on x86_64
+ running on linux/amd64 with uid 1000 gid 1000
+ effective caps 0x0000000000000000 -chown -dac_override -dac_read_search ...
+```
+
+### JSON Output
+Display version information in JSON format:
+```bash
+$ subtrace version -json
+{
+ "release": "b123",
+ "commitHash": "abc123def",
+ "commitTime": "2024-03-15T10:30:00Z",
+ "buildTime": "2024-03-15T10:30:00Z",
+ "buildGoVersion": "go1.22.0",
+ "buildOS": "linux",
+ "buildArch": "amd64",
+ "executableHash": "8a7b6c5d4e3f...",
+ "kernelName": "Linux",
+ "kernelVersion": "5.15.0",
+ "kernelArch": "x86_64",
+ "uid": 1000,
+ "gid": 1000,
+ "effectiveCaps": "0x0000000000000000 -chown -dac_override ..."
+}
+```
+
+## Use Cases
+
+The version command is commonly used for troubleshooting purposes:
+- Checking which version of Subtrace is installed
+- Gathering system information for bug reports
+- Verifying the build environment and capabilities
+- Getting JSON-formatted version data for programmatic use
+- Troubleshooting version-specific issues
+- Verifying system compatibility
+
+## Notes
+
+- The executable hash is calculated from the first 64MB of the binary
+- Capability flags are only shown on Linux systems
+- JSON output can be particularly useful for integration with other tools
+- All timestamps are in UTC
+
+## Related Commands
+
+- [`subtrace run`](/references/subtrace-run) - Run a process with tracing enabled
+- [`subtrace proxy`](/references/subtrace-proxy) - Start a reverse proxy with tracing
+- [`subtrace worker`](/references/subtrace-worker) - Start a worker node (for self-hosted setups)
diff --git a/references/subtrace-worker.mdx b/references/subtrace-worker.mdx
new file mode 100644
index 0000000..a8a8f1e
--- /dev/null
+++ b/references/subtrace-worker.mdx
@@ -0,0 +1,131 @@
+---
+title: 'subtrace worker'
+description: 'The worker command starts a Subtrace worker node that processes and stores trace data in ClickHouse.'
+---
+
+## Synopsis
+
+```bash
+subtrace worker [flags]
+```
+
+## Description
+
+The `subtrace worker` command starts a worker node that connects to ClickHouse for storing and processing trace data. The worker is responsible for:
+- Applying ClickHouse migrations
+- Handling data tunnels
+- Processing and storing trace events
+- Managing event schemas
+
+## Options
+
+| Flag | Description | Default | Required |
+|------|-------------|---------|-----------|
+| -clickhouse-host | ClickHouse server hostname | localhost | No |
+| -clickhouse-database | ClickHouse database name | subtrace | No |
+| -v | Enable verbose debug logging | false | No |
+
+## Environment Variables
+
+| Variable | Description | Required |
+|----------|-------------|-----------|
+| SUBTRACE_TOKEN | Authentication token for the Subtrace service | Yes |
+| SUBTRACE_ENDPOINT | Custom API endpoint (e.g., for self-hosted deployments) | No |
+| SUBTRACE_CLICKHOUSE_SEED_NAMESPACE | UUID for seeding initial data | No |
+
+## Examples
+
+### Basic Usage
+
+Start a worker with default settings:
+```bash
+$ export SUBTRACE_TOKEN=your_token_here
+$ subtrace worker
+```
+
+### Custom ClickHouse Configuration
+
+Connect to a specific ClickHouse instance:
+```bash
+$ subtrace worker \
+ -clickhouse-host clickhouse.example.com \
+ -clickhouse-database traces
+```
+
+### Debug Mode
+
+Run with verbose logging enabled:
+```bash
+$ subtrace worker -v
+```
+
+### Docker Usage
+
+Run worker in a Docker container with built-in ClickHouse:
+```bash
+$ docker run -e SUBTRACE_TOKEN=your_token_here subtrace/worker
+```
+
+## Exit Codes
+
+| Code | Description |
+|------|-------------|
+| 0 | Successful execution |
+| 1 | General error (configuration, connection, etc.) |
+
+## Operational Notes
+
+- The worker requires a valid SUBTRACE_TOKEN for authentication
+- ClickHouse connection is retried up to 5 times with exponential backoff
+- Database migrations are automatically applied on startup
+- Worker supports automatic schema evolution as new event types are received
+- Multiple workers can be run in parallel for horizontal scaling
+- The worker maintains a persistent connection to the Subtrace service for real-time event processing
+
+## Data Schema
+
+The worker creates and maintains the following ClickHouse tables:
+- `subtrace_events_*` - Main event storage tables
+- `subtrace_ingest_*` - Temporary tables for data ingestion
+- `subtrace_migrations_*` - Schema migration tracking
+
+## Common Tasks
+
+### Verifying Worker Status
+```bash
+# Check worker logs
+$ subtrace worker -v 2>&1 | grep "starting worker node"
+
+# Verify ClickHouse connection
+$ subtrace worker -v 2>&1 | grep "connected to clickhouse server"
+```
+
+### Monitoring Event Processing
+```bash
+# Watch real-time processing with verbose logging
+$ subtrace worker -v 2>&1 | grep "tunnel"
+```
+
+### Troubleshooting
+
+If the worker fails to start:
+1. Verify SUBTRACE_TOKEN is set correctly
+2. Ensure ClickHouse is accessible
+3. Check for sufficient disk space
+4. Verify network connectivity
+5. Enable verbose logging with -v flag
+
+### Production Deployment
+
+For production environments:
+1. Use Docker for consistent deployment
+2. Configure resource limits
+3. Set up monitoring
+4. Use separate ClickHouse instance
+5. Enable automated restarts
+
+## Related Commands
+
+- [`subtrace run`](/references/subtrace-run) - Run a process with tracing enabled
+- [`subtrace proxy`](/references/subtrace-proxy) - Start a reverse proxy with tracing
+- [`subtrace version`](/references/subtrace-version) - Show version information
diff --git a/troubleshooting-guide.mdx b/troubleshooting-guide.mdx
new file mode 100644
index 0000000..064a742
--- /dev/null
+++ b/troubleshooting-guide.mdx
@@ -0,0 +1,219 @@
+---
+title: "Troubleshooting FAQs"
+description: "Common issues, solutions, and root causes when working with Subtrace"
+lastUpdated: "2024-12-16"
+---
+
+# Troubleshooting Guide
+
+This guide covers common issues you may encounter when using Subtrace and provides detailed solutions. If you can't find a solution to your problem, please reach out on [Discord](https://subtrace.dev/discord).
+
+## TLS/Certificate Issues
+
+### Symptoms
+- TLS / SSL handshake failures when running your services
+- Unable to get local issuer certificate
+- Max retries exceeded when connecting to via SSL
+- Applications failing to connect to HTTPS endpoints
+- Error messages containing "unknown certificate authority" or "remote error: tls: unknown certificate authority"
+
+### Root Cause
+Subtrace intercepts TLS traffic by creating an ephemeral CA certificate on your machine. Some applications and languages (like Erlang, Rust) use their own certificate stores and don't recognize this CA.
+
+### Solutions
+
+1. Disable TLS interception temporarily:
+```bash
+subtrace run -tls=false -- your_application
+```
+
+2. For Erlang/Elixir applications, configure OpenSSL:
+```bash
+ELIXIR_ERL_OPTIONS="+fnu" subtrace run -- your_application
+```
+
+## No traces in my dashboard
+
+### Symptoms
+
+- No traces appearing in dashboard, even though code runs properly
+- Connection errors to worker nodes
+- Missing traces
+- Warning that `subtrace proxy was started with -log=false but SUBTRACE_TOKEN is empty`
+
+### Root Cause
+Worker nodes need proper configuration and AuthN / AuthZ.
+
+- The docker image is not running
+- The docker image is running, but your subtrace worker is not authenticated
+- Your subtrace proxy is not authorized to send traces to your workspace
+- Your subtrace proxy has not found a tracer token in your environment
+- You did not run your subtrace command properly.
+
+### Solutions
+
+1. Check that you can run subtrace properly from your `$PATH`:
+
+```bash
+subtrace version -json
+```
+
+2. Make sure your docker image is running:
+
+```bash
+docker ps
+```
+
+3. Restart your docker image:
+
+```bash
+docker restart your_image_name
+```
+
+4. Reset your **worker token** in the [Tokens page](https://subtrace.dev/dashboard/tokens), then re-authenticate your subtrace worker:
+
+```bash
+docker run -d \
+ -e SUBTRACE_TOKEN=subt_REDACTED_WORKER_TOKEN \
+ -v subtrace:/var/lib/clickhouse \
+ subtrace.dev/worker:latest
+```
+
+5. Make sure you set `SUBTRACE_TOKEN` in your environment. Otherwise, your subtrace proxy will not be able to send traces to your workspace:
+
+```bash
+export SUBTRACE_TOKEN=subt_YOUR_TOKEN_HERE
+```
+
+
+You can also reset your **worker token** in the [Tokens page](https://subtrace.dev/dashboard/tokens)
+
+
+6. If self-hosting Clickhouse:
+ - Check Clickhouse is running: `curl http://localhost:8123`
+ - Verify database exists and is accessible
+ - Check worker logs for connection errors
+
+7. Check your network connectivity:
+ - Ensure firewall rules allow worker connections
+ - Verify DNS resolution for Clickhouse hosts
+ - Check network latency between components
+
+## Path/Binary Not Found
+
+### Symptoms
+- Docker container fails to start
+- Error messages about missing subtrace executable
+- "subtrace: executable file not found in $PATH" errors
+
+### Root Cause
+The subtrace binary must be included in the container and added to the PATH.
+
+### Solutions
+
+1. Add subtrace to your Dockerfile:
+```dockerfile
+COPY --from=subtrace/subtrace /usr/local/bin/subtrace /usr/local/bin/subtrace
+```
+
+2. If installing locally, ensure the binary is in your PATH:
+```bash
+# Add to ~/.bashrc or ~/.zshrc
+export PATH=$PATH:/path/to/subtrace
+```
+
+## Authentication Errors
+
+### Symptoms
+- "Failed to initialize" errors
+- Token-related errors
+- Unable to connect to Subtrace services
+
+### Root Cause
+Using the wrong token type or missing token environment variable.
+
+### Solutions
+
+1. Use the correct token type:
+ - Tracer tokens: For running applications
+ - Worker tokens: For worker nodes
+
+2. Set the token environment variable:
+```bash
+export SUBTRACE_TOKEN=subt_your_token_here
+subtrace run -- your_application
+```
+
+3. Generate a new token:
+ - Visit https://subtrace.dev/dashboard
+ - Navigate to Tokens section
+ - Click "Create tracer token"
+
+## Performance Impact
+
+### Symptoms
+- Increased latency in applications
+- Performance concerns
+- Higher resource usage
+
+### Root Cause
+Subtrace adds monitoring overhead, though it's designed to be minimal (less than 0.1ms in typical cases).
+
+### Solutions
+
+1. Configure selective tracing using JavaScript:
+```javascript
+trace("POST", "/api/login", (req, resp) => {
+ // Only trace specific endpoints
+});
+```
+
+2. Adjust payload limits:
+```bash
+subtrace run --payload-limit=1024 -- your_application
+```
+
+3. Filter noisy endpoints:
+```javascript
+trace("GET", "/health", () => false); // Skip health checks
+```
+
+## Common Error Messages
+
+### "failed to create task for container"
+This usually indicates the subtrace binary isn't properly installed in your container. Add it to your Dockerfile as shown in the Path/Binary Not Found section.
+
+### "tls: unknown certificate authority"
+This occurs when applications don't trust Subtrace's ephemeral CA. Either configure the application to use the system certificate store or disable TLS interception with `-tls=false`.
+
+### "SUBTRACE_TOKEN is empty"
+Set your token as an environment variable:
+```bash
+export SUBTRACE_TOKEN=subt_your_token_here
+```
+
+## Getting Help
+
+If you encounter issues not covered in this guide:
+
+1. Check [documentation](https://docs.subtrace.dev)
+2. Join our [Discord community](https://subtrace.dev/discord)
+3. Review [GitHub Issues](https://github.com/subtrace/subtrace/issues)
+4. Contact support through [dashboard](https://subtrace.dev/dashboard)
+
+When reporting issues, please include:
+- Subtrace version (`subtrace version`)
+- Operating system and version
+- Complete error message
+- Steps to reproduce
+- Relevant application logs
+
+## Feature Status
+
+Some features are still in development:
+
+- HTTP/2 support is being worked on ([Issue #4](https://github.com/subtrace/subtrace/issues/4))
+- Cross-service tracing is planned for future releases
+- Case-sensitive column handling improvements are in progress
+
+Check our GitHub repository for the latest updates and planned features.