22 lines
864 B
Bash
22 lines
864 B
Bash
#!/bin/bash
|
|
# Run on supersrv: bash ingest.sh
|
|
# Requires: Docker with ClickHouse container avs3owrcdr53mqyun9s910mb
|
|
|
|
set -e
|
|
CONTAINER="avs3owrcdr53mqyun9s910mb"
|
|
DATA_FILE="$(dirname "$0")/knowledge-base.jsonl"
|
|
|
|
echo "=== Creating money_raise database and table ==="
|
|
docker exec -i "$CONTAINER" clickhouse-client --multiquery < "$(dirname "$0")/schema.sql"
|
|
|
|
echo "=== Ingesting knowledge base ($(wc -l < "$DATA_FILE") entries) ==="
|
|
docker exec -i "$CONTAINER" clickhouse-client \
|
|
--query "INSERT INTO money_raise.knowledge_base \
|
|
(category, subcategory, region, stage, title, content, format, tags, source, language) \
|
|
FORMAT JSONEachRow" < "$DATA_FILE"
|
|
|
|
echo "=== Verifying ==="
|
|
docker exec "$CONTAINER" clickhouse-client \
|
|
--query "SELECT category, count() as n FROM money_raise.knowledge_base GROUP BY category ORDER BY n DESC"
|
|
|
|
echo "=== Done! ==="
|