Mirror of https://github.com/roostorg/osprey
github.com/roostorg/osprey
1# Volumes for druid purposes only
2volumes:
3 metadata_data: {}
4 middle_var: {}
5 historical_var: {}
6 broker_var: {}
7 coordinator_var: {}
8 router_var: {}
9 druid_shared: {}
10 minio_data: {}
11
12services:
13 osprey-kafka:
14 image: confluentinc/cp-kafka:7.4.0
15 hostname: osprey-kafka
16 container_name: osprey-kafka
17 ports:
18 - "127.0.0.1:9092:9092"
19 environment:
20 KAFKA_NODE_ID: 1
21 KAFKA_PROCESS_ROLES: "broker,controller"
22 KAFKA_CONTROLLER_QUORUM_VOTERS: "1@osprey-kafka:29093"
23 KAFKA_CONTROLLER_LISTENER_NAMES: "CONTROLLER"
24 KAFKA_INTER_BROKER_LISTENER_NAME: "INTERNAL"
25 KAFKA_LISTENERS: "INTERNAL://osprey-kafka:29092,EXTERNAL://0.0.0.0:9092,CONTROLLER://osprey-kafka:29093"
26 KAFKA_ADVERTISED_LISTENERS: "INTERNAL://osprey-kafka:29092,EXTERNAL://localhost:9092"
27 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT"
28 KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
29 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
30 KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
31 KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
32 CLUSTER_ID: "P45WxmmWSe2CrdGoeJMcKg"
33 healthcheck:
34 test:
35 [
36 "CMD",
37 "bash",
38 "-c",
39 "kafka-topics --bootstrap-server osprey-kafka:29092 --list",
40 ]
41 interval: 10s
42 timeout: 5s
43 retries: 5
44
45 minio:
46 image: minio/minio:latest
47 container_name: minio
48 hostname: minio
49 ports:
50 - "127.0.0.1:9000:9000" # minio API
51 - "127.0.0.1:9001:9001" # minio Console
52 environment:
53 MINIO_ROOT_USER: minioadmin
54 MINIO_ROOT_PASSWORD: minioadmin123
55 volumes:
56 - minio_data:/data
57 command: server --console-address ":9001" /data
58 healthcheck:
59 test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
60 interval: 10s
61 timeout: 5s
62 retries: 3
63
64 minio-bucket-init:
65 image: minio/mc:latest
66 depends_on:
67 minio:
68 condition: service_healthy
69 entrypoint: ["/bin/sh", "/init-minio-bucket.sh"]
70 volumes:
71 - ./init-minio-bucket.sh:/init-minio-bucket.sh
72 restart: "no"
73
74 osprey-kafka-topic-creator:
75 image: confluentinc/cp-kafka:7.4.0
76 depends_on:
77 osprey-kafka:
78 condition: service_healthy
79 command: >
80 bash -c "
81 kafka-topics --bootstrap-server osprey-kafka:29092 --create --if-not-exists --topic osprey.actions_input --partitions 3 --replication-factor 1 &&
82 kafka-topics --bootstrap-server osprey-kafka:29092 --create --if-not-exists --topic osprey.execution_results --partitions 3 --replication-factor 1 &&
83 kafka-topics --bootstrap-server osprey-kafka:29092 --list
84 "
85
86 osprey-worker:
87 container_name: osprey-worker
88 hostname: osprey-worker
89 build:
90 context: .
91 dockerfile: osprey_worker/Dockerfile
92 depends_on:
93 osprey-kafka:
94 condition: service_healthy
95 osprey-kafka-topic-creator:
96 condition: service_completed_successfully
97 minio:
98 condition: service_healthy
99 minio-bucket-init:
100 condition: service_completed_successfully
101 ports:
102 - "127.0.0.1:5001:5000"
103 command: ["osprey-worker"]
104 environment:
105 - PYTHONPATH=/osprey
106 - PORT=5000
107 - POSTGRES_HOSTS={"osprey_db":"postgresql://osprey:FoolishPassword@postgres:5432/osprey"}
108 - OSPREY_INPUT_STREAM_SOURCE=kafka
109 - OSPREY_STDOUT_OUTPUT_SINK=True
110 - OSPREY_KAFKA_BOOTSTRAP_SERVERS=["osprey-kafka:29092"]
111 - OSPREY_KAFKA_INPUT_STREAM_TOPIC=osprey.actions_input
112 # Client ID will default to the machine hostname if it isn't defined
113 - OSPREY_KAFKA_INPUT_STREAM_CLIENT_ID=localhost
114 - OSPREY_KAFKA_OUTPUT_SINK=True
115 - OSPREY_KAFKA_OUTPUT_TOPIC=osprey.execution_results
116 - OSPREY_KAFKA_OUTPUT_CLIENT_ID=localhost
117 - DD_TRACE_ENABLED=False
118 - DD_DOGSTATSD_DISABLE=True
119 - OSPREY_RULES_SINK_NUM_WORKERS=1
120 - BIGTABLE_EMULATOR_HOST=bigtable:8361
121 - OSPREY_EXECUTION_RESULT_STORAGE_BACKEND=minio
122 - OSPREY_MINIO_ENDPOINT=minio:9000
123 - OSPREY_MINIO_ACCESS_KEY=minioadmin
124 - OSPREY_MINIO_SECRET_KEY=minioadmin123
125 - OSPREY_MINIO_SECURE=false
126 - OSPREY_MINIO_EXECUTION_RESULTS_BUCKET=execution-output
127 - SNOWFLAKE_API_ENDPOINT=http://snowflake-id-worker:8088
128 - OSPREY_RULES_PATH=./example_rules
129 volumes:
130 - ./osprey_worker:/osprey/osprey_worker
131 - ./osprey_rpc:/osprey/osprey_rpc
132 - ./example_rules:/osprey/example_rules
133 - ./entrypoint.sh:/osprey/entrypoint.sh
134
135 osprey-ui-api:
136 container_name: osprey-ui-api
137 build:
138 context: .
139 dockerfile: osprey_worker/Dockerfile
140 depends_on:
141 - osprey-worker
142 - druid-broker
143 - postgres
144 - snowflake-id-worker
145 ports:
146 - "127.0.0.1:5004:5004"
147 command: ["osprey-ui-api"]
148 environment:
149 - PYTHONPATH=/osprey
150 - PORT=5004
151 - DEBUG=true
152 - FLASK_DEBUG=1
153 - FLASK_ENV=development
154 - DRUID_URL=http://druid-broker:8082
155 - POSTGRES_HOSTS={"osprey_db":"postgresql://osprey:FoolishPassword@postgres:5432/osprey"}
156 - DD_TRACE_ENABLED=False
157 - DD_DOGSTATSD_DISABLE=True
158 - OSPREY_RULES_PATH=/osprey/example_rules
159 - OSPREY_DISABLE_VALIDATION_EXPORTER=true
160 - BIGTABLE_EMULATOR_HOST=bigtable:8361
161 - SNOWFLAKE_API_ENDPOINT=http://snowflake-id-worker:8088
162 - SNOWFLAKE_EPOCH=1420070400000
163 volumes:
164 - ./osprey_worker:/osprey/osprey_worker
165 - ./osprey_rpc:/osprey/osprey_rpc
166 - ./example_rules:/osprey/example_rules
167
168 osprey-ui:
169 container_name: osprey-ui
170 hostname: osprey-ui
171 build:
172 context: .
173 dockerfile: osprey_ui/Dockerfile
174 depends_on:
175 - osprey-ui-api
176 ports:
177 - "127.0.0.1:5002:5002"
178 environment:
179 - NODE_ENV=development
180 - REACT_APP_API_BASE_URL=http://localhost:5004
181 volumes:
182 - ./osprey_ui:/app
183 - /app/node_modules
184
185 snowflake-id-worker:
186 hostname: snowflake-id-worker
187 container_name: snowflake-id-worker
188 image: ghcr.io/ayubun/snowflake-id-worker:0
189 ports:
190 - "127.0.0.1:8088:8088"
191 environment:
192 - WORKER_ID=0
193 - DATA_CENTER_ID=0
194 - EPOCH=1420070400000
195 - PORT=8088
196 restart: unless-stopped
197
198 bigtable:
199 hostname: bigtable
200 container_name: bigtable
201 image: gcr.io/google.com/cloudsdktool/cloud-sdk:latest
202 ports:
203 - "127.0.0.1:8361:8361"
204 command: >
205 bash -c "
206 gcloud beta emulators bigtable start --host-port=0.0.0.0:8361 --project=osprey-dev
207 "
208 healthcheck:
209 test: ["CMD", "bash", "-c", "pgrep -f cbtemulator > /dev/null || exit 1"]
210 interval: 10s
211 timeout: 5s
212 retries: 5
213 restart: unless-stopped
214
215 bigtable-initializer:
216 container_name: bigtable-initializer
217 image: gcr.io/google.com/cloudsdktool/cloud-sdk:latest
218 depends_on:
219 bigtable:
220 condition: service_healthy
221 volumes:
222 - ./init-bigtable.sh:/init-bigtable.sh
223 command: ["/bin/bash", "/init-bigtable.sh"]
224
225 # Optional test data generator - run with:
226 # docker compose --profile test_data up osprey-kafka-test-data-producer -d
227 osprey-kafka-test-data-producer:
228 image: confluentinc/cp-kafka:7.4.0
229 hostname: osprey-kafka-test-data-producer
230 container_name: osprey-kafka-test-data-producer
231 depends_on:
232 osprey-kafka:
233 condition: service_healthy
234 osprey-kafka-topic-creator:
235 condition: service_completed_successfully
236 profiles:
237 - test_data
238 - test-data
239 environment:
240 KAFKA_TOPIC: "osprey.actions_input"
241 KAFKA_BROKER: "osprey-kafka:29092"
242 volumes:
243 - ./example_data:/osprey/example_data
244 entrypoint:
245 - /bin/bash
246 command: ["/osprey/example_data/generate_test_data.sh"]
247
248 postgres:
249 hostname: postgres
250 container_name: postgres
251 image: postgres:18
252 ports:
253 - "127.0.0.1:5432:5432"
254 volumes:
255 - metadata_data:/var/lib/postgresql
256 environment:
257 - POSTGRES_PASSWORD=FoolishPassword
258 - POSTGRES_USER=osprey
259 - POSTGRES_DB=osprey
260 healthcheck:
261 test: pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB
262 start_period: 30s
263 interval: 10s
264 timeout: 10s
265 retries: 5
266
267 # DRUID, HERE BE DRAGONS
268 # Need 3.5 or later for container nodes
269 druid-zookeeper:
270 hostname: druid-zookeeper
271 container_name: druid-zookeeper
272 image: zookeeper:3.5.10
273 ports:
274 - "127.0.0.1:2181:2181"
275 environment:
276 - ZOO_MY_ID=1
277
278 druid-coordinator:
279 image: apache/druid:34.0.0
280 hostname: druid-coordinator
281 container_name: druid-coordinator
282 volumes:
283 - druid_shared:/opt/shared
284 - coordinator_var:/opt/druid/var
285 depends_on:
286 - druid-zookeeper
287 - postgres
288 ports:
289 - "127.0.0.1:8081:8081"
290 command:
291 - coordinator
292 env_file:
293 - druid/environment
294
295 druid-broker:
296 image: apache/druid:34.0.0
297 container_name: druid-broker
298 hostname: druid-broker
299 volumes:
300 - broker_var:/opt/druid/var
301 depends_on:
302 - druid-zookeeper
303 - postgres
304 - druid-coordinator
305 ports:
306 - "127.0.0.1:8082:8082"
307 command:
308 - broker
309 env_file:
310 - druid/environment
311
312 druid-historical:
313 image: apache/druid:34.0.0
314 container_name: druid-historical
315 hostname: druid-historical
316 volumes:
317 - druid_shared:/opt/shared
318 - historical_var:/opt/druid/var
319 depends_on:
320 - druid-zookeeper
321 - postgres
322 - druid-coordinator
323 ports:
324 - "127.0.0.1:8083:8083"
325 command:
326 - historical
327 env_file:
328 - druid/environment
329
330 druid-middlemanager:
331 image: apache/druid:34.0.0
332 container_name: druid-middlemanager
333 hostname: druid-middlemanager
334 volumes:
335 - druid_shared:/opt/shared
336 - middle_var:/opt/druid/var
337 depends_on:
338 - druid-zookeeper
339 - postgres
340 - druid-coordinator
341 ports:
342 - "127.0.0.1:8091:8091"
343 - "127.0.0.1:8100-8105:8100-8105"
344 command:
345 - middleManager
346 env_file:
347 - druid/environment
348
349 druid-router:
350 image: apache/druid:34.0.0
351 container_name: druid-router
352 hostname: druid-router
353 volumes:
354 - router_var:/opt/druid/var
355 depends_on:
356 - druid-zookeeper
357 - postgres
358 - druid-coordinator
359 ports:
360 - "127.0.0.1:8888:8888"
361 command:
362 - router
363 env_file:
364 - druid/environment
365
366 druid-spec-submitter:
367 image: curlimages/curl:latest
368 depends_on:
369 - druid-coordinator
370 volumes:
371 - ./druid/specs:/specs
372 command: ["/bin/sh", "/specs/submit-specs.sh"]
373 restart: "no"