From b1112a4b26d1fbaf17ebe6bdbee3ae92eebbd592 Mon Sep 17 00:00:00 2001 From: Felix Mann Date: Fri, 19 Feb 2016 10:31:18 +0100 Subject: [PATCH] Created folder for InfluxDB 0.8.9 Use Influx 0.8.9 instead of 0.8.8 so that we can export data to 0.9/0.10 later on. Corrected the URL for 0.8.9. Removed empty lines in config section graphite Replacement of variable GRAPHITE_TEMPLATE with sed command doesn't work if there are empty lines within the config section graphite Fix cli path in README Add a docker file for 0.10.3 that is literally copied from 0.10. Removed 0.10.3 folder, applied change to 0.10/Dockerfile instead. --- 0.10/config.toml | 2 - 0.8.9/Dockerfile | 36 ++++++++++ 0.8.9/config.toml | 178 ++++++++++++++++++++++++++++++++++++++++++++++ 0.8.9/run.sh | 112 +++++++++++++++++++++++++++++ README.md | 7 +- 5 files changed, 330 insertions(+), 5 deletions(-) create mode 100644 0.8.9/Dockerfile create mode 100644 0.8.9/config.toml create mode 100755 0.8.9/run.sh diff --git a/0.10/config.toml b/0.10/config.toml index e34513c..b644cf3 100644 --- a/0.10/config.toml +++ b/0.10/config.toml @@ -157,10 +157,8 @@ reporting-disabled = false templates = [ # filter + template #"*.app env.service.resource.measurement", - # filter + template + extra tag #"stats.* .host.measurement* region=us-west,agent=sensu", - # default template. Ignore the first graphite component "servers" "instance.profile.measurement*" ] diff --git a/0.8.9/Dockerfile b/0.8.9/Dockerfile new file mode 100644 index 0000000..0bd63dd --- /dev/null +++ b/0.8.9/Dockerfile @@ -0,0 +1,36 @@ +FROM tutum/curl:trusty +MAINTAINER Feng Honglin + +# Install InfluxDB +ENV INFLUXDB_VERSION 0.8.9 +RUN curl -s -o /tmp/influxdb_latest_amd64.deb http://get.influxdb.org.s3.amazonaws.com/influxdb_${INFLUXDB_VERSION}_amd64.deb && \ + dpkg -i /tmp/influxdb_latest_amd64.deb && \ + rm /tmp/influxdb_latest_amd64.deb && \ + rm -rf /var/lib/apt/lists/* + +ADD config.toml /config/config.toml +ADD run.sh /run.sh +RUN chmod +x /*.sh + +ENV PRE_CREATE_DB **None** +ENV SSL_SUPPORT **False** +ENV SSL_CERT **None** + +# Admin server +EXPOSE 8083 + +# HTTP API +EXPOSE 8086 + +# HTTPS API +EXPOSE 8084 + +# Raft port (for clustering, don't expose publicly!) +#EXPOSE 8090 + +# Protobuf port (for clustering, don't expose publicly!) +#EXPOSE 8099 + +VOLUME ["/data"] + +CMD ["/run.sh"] diff --git a/0.8.9/config.toml b/0.8.9/config.toml new file mode 100644 index 0000000..0aea43c --- /dev/null +++ b/0.8.9/config.toml @@ -0,0 +1,178 @@ +# Welcome to the InfluxDB configuration file. + +# If hostname (on the OS) doesn't return a name that can be resolved by the other +# systems in the cluster, you'll have to set the hostname to an IP or something +# that can be resolved here. +# hostname = "" + +bind-address = "0.0.0.0" + +[logging] +# logging level can be one of "debug", "info", "warn" or "error" +level = "info" +file = "/opt/influxdb/shared/log.txt" # stdout to log to standard out + +# Configure the admin server +[admin] +port = 8083 # binding is disabled if the port isn't set +assets = "/opt/influxdb/current/admin" + +# Configure the http api +[api] +port = 8086 # binding is disabled if the port isn't set +# ssl-port = 8084 # Ssl support is enabled if you set a port and cert +# ssl-cert = "/cert.pem" + +# connections will timeout after this amount of time. Ensures that clients that misbehave +# and keep alive connections they don't use won't end up connection a million times. +# However, if a request is taking longer than this to complete, could be a problem. +read-timeout = "5s" + +[input_plugins] + + # Configure the graphite api + [input_plugins.graphite] + enabled = false + # port = 2003 + # database = "" # store graphite data in this database + # udp_enabled = true # enable udp interface on the same port as the tcp interface + + [input_plugins.udp] + enabled = false + # port = 4444 + # database = "" + +# Raft configuration +[raft] +# The raft port should be open between all servers in a cluster. +# However, this port shouldn't be accessible from the internet. + +port = 8090 + +# Where the raft logs are stored. The user running InfluxDB will need read/write access. +dir = "/data/raft" + +# election-timeout = "1s" + +[storage] +dir = "/data/db" +# How many requests to potentially buffer in memory. If the buffer gets filled then writes +# will still be logged and once the local storage has caught up (or compacted) the writes +# will be replayed from the WAL +write-buffer-size = 10000 + +[cluster] +# A comma separated list of servers to seed +# this server. this is only relevant when the +# server is joining a new cluster. Otherwise +# the server will use the list of known servers +# prior to shutting down. Any server can be pointed to +# as a seed. It will find the Raft leader automatically. + +# Here's an example. Note that the port on the host is the same as the raft port. +# seed-servers = ["hosta:8090","hostb:8090"] + +# Replication happens over a TCP connection with a Protobuf protocol. +# This port should be reachable between all servers in a cluster. +# However, this port shouldn't be accessible from the internet. + +protobuf_port = 8099 +protobuf_timeout = "2s" # the write timeout on the protobuf conn any duration parseable by time.ParseDuration +protobuf_heartbeat = "200ms" # the heartbeat interval between the servers. must be parseable by time.ParseDuration +protobuf_min_backoff = "1s" # the minimum backoff after a failed heartbeat attempt +protobuf_max_backoff = "10s" # the maxmimum backoff after a failed heartbeat attempt + +# How many write requests to potentially buffer in memory per server. If the buffer gets filled then writes +# will still be logged and once the server has caught up (or come back online) the writes +# will be replayed from the WAL +write-buffer-size = 10000 + +# the maximum number of responses to buffer from remote nodes, if the +# expected number of responses exceed this number then querying will +# happen sequentially and the buffer size will be limited to this +# number +max-response-buffer-size = 100 + +# When queries get distributed out to shards, they go in parallel. This means that results can get buffered +# in memory since results will come in any order, but have to be processed in the correct time order. +# Setting this higher will give better performance, but you'll need more memory. Setting this to 1 will ensure +# that you don't need to buffer in memory, but you won't get the best performance. +concurrent-shard-query-limit = 10 + +[leveldb] + +# Maximum mmap open files, this will affect the virtual memory used by +# the process +max-open-files = 40 + +# LRU cache size, LRU is used by leveldb to store contents of the +# uncompressed sstables. You can use `m` or `g` prefix for megabytes +# and gigabytes, respectively. +lru-cache-size = "200m" + +# The default setting on this is 0, which means unlimited. Set this to something if you want to +# limit the max number of open files. max-open-files is per shard so this * that will be max. +max-open-shards = 0 + +# The default setting is 100. This option tells how many points will be fetched from LevelDb before +# they get flushed into backend. +point-batch-size = 100 + +# The number of points to batch in memory before writing them to leveldb. Lowering this number will +# reduce the memory usage, but will result in slower writes. +write-batch-size = 5000000 + +# These options specify how data is sharded across the cluster. There are two +# shard configurations that have the same knobs: short term and long term. +# Any series that begins with a capital letter like Exceptions will be written +# into the long term storage. Any series beginning with a lower case letter +# like exceptions will be written into short term. The idea being that you +# can write high precision data into short term and drop it after a couple +# of days. Meanwhile, continuous queries can run downsampling on the short term +# data and write into the long term area. +[sharding] + # how many servers in the cluster should have a copy of each shard. + # this will give you high availability and scalability on queries + replication-factor = 1 + + [sharding.short-term] + # each shard will have this period of time. Note that it's best to have + # group by time() intervals on all queries be < than this setting. If they are + # then the aggregate is calculated locally. Otherwise, all that data gets sent + # over the network when doing a query. + duration = "7d" + + # split will determine how many shards to split each duration into. For example, + # if we created a shard for 2014-02-10 and split was set to 2. Then two shards + # would be created that have the data for 2014-02-10. By default, data will + # be split into those two shards deterministically by hashing the (database, serise) + # tuple. That means that data for a given series will be written to a single shard + # making querying efficient. That can be overridden with the next option. + split = 1 + + # You can override the split behavior to have the data for series that match a + # given regex be randomly distributed across the shards for a given interval. + # You can use this if you have a hot spot for a given time series writing more + # data than a single server can handle. Most people won't have to resort to this + # option. Also note that using this option means that queries will have to send + # all data over the network so they won't be as efficient. + # split-random = "/^hf.*/" + + [sharding.long-term] + duration = "30d" + split = 1 + # split-random = "/^Hf.*/" + +[wal] + +dir = "/data/wal" +flush-after = 1000 # the number of writes after which wal will be flushed, 0 for flushing on every write +bookmark-after = 1000 # the number of writes after which a bookmark will be created + +# the number of writes after which an index entry is created pointing +# to the offset of the first request, default to 1k +index-after = 1000 + +# the number of requests per one log file, if new requests came in a +# new log file will be created +requests-per-logfile = 10000 diff --git a/0.8.9/run.sh b/0.8.9/run.sh new file mode 100755 index 0000000..248f268 --- /dev/null +++ b/0.8.9/run.sh @@ -0,0 +1,112 @@ +#!/bin/bash + +set -m +CONFIG_FILE="/config/config.toml" + +# Dynamically change the value of 'max-open-shards' to what 'ulimit -n' returns +sed -i "s/^max-open-shards.*/max-open-shards = $(ulimit -n)/" ${CONFIG_FILE} + +# Configure InfluxDB Cluster +if [ -n "${FORCE_HOSTNAME}" ]; then + if [ "${FORCE_HOSTNAME}" == "auto" ]; then + #set hostname with IPv4 eth0 + HOSTIPNAME=$(ip a show dev eth0 | grep inet | grep eth0 | sed -e 's/^.*inet.//g' -e 's/\/.*$//g') + /usr/bin/perl -p -i -e "s/^# hostname.*$/hostname = \"${HOSTIPNAME}\"/g" ${CONFIG_FILE} + else + /usr/bin/perl -p -i -e "s/^# hostname.*$/hostname = \"${FORCE_HOSTNAME}\"/g" ${CONFIG_FILE} + fi +fi + +if [ -n "${SEEDS}" ]; then + SEEDS=$(eval SEEDS=$SEEDS ; echo $SEEDS | grep '^\".*\"$' || echo "\""$SEEDS"\"" | sed -e 's/, */", "/g') + /usr/bin/perl -p -i -e "s/^# seed-servers.*$/seed-servers = [${SEEDS}]/g" ${CONFIG_FILE} +fi + +if [ -n "${REPLI_FACTOR}" ]; then + /usr/bin/perl -p -i -e "s/replication-factor = 1/replication-factor = ${REPLI_FACTOR}/g" ${CONFIG_FILE} +fi + +if [ "${PRE_CREATE_DB}" == "**None**" ]; then + unset PRE_CREATE_DB +fi + +if [ "${SSL_CERT}" == "**None**" ]; then + unset SSL_CERT +fi + +if [ "${SSL_SUPPORT}" == "**False**" ]; then + unset SSL_SUPPORT +fi + +# Add UDP support +if [ -n "${UDP_DB}" ]; then + sed -i -r -e "/^\s+\[input_plugins.udp\]/, /^$/ { s/false/true/; s/#//g; s/\"\"/\"${UDP_DB}\"/g; }" ${CONFIG_FILE} +fi +if [ -n "${UDP_PORT}" ]; then + sed -i -r -e "/^\s+\[input_plugins.udp\]/, /^$/ { s/4444/${UDP_PORT}/; }" ${CONFIG_FILE} +fi + +# SSL SUPPORT (Enable https support on port 8084) +API_URL="http://localhost:8086" +CERT_PEM="/cert.pem" +SUBJECT_STRING="/C=US/ST=NewYork/L=NYC/O=Tutum/CN=*" +if [ -n "${SSL_SUPPORT}" ]; then + echo "=> SSL Support enabled, using SSl api ..." + echo "=> Listening on port 8084(https api), disabling port 8086(http api)" + if [ -n "${SSL_CERT}" ]; then + echo "=> Use user uploaded certificate" + echo -e "${SSL_CERT}" > ${CERT_PEM} + else + echo "=> Use self-signed certificate" + if [ -f ${CERT_PEM} ]; then + echo "=> Certificate found, skip ..." + else + echo "=> Generating certificate ..." + openssl req -new -newkey rsa:4096 -days 3650 -nodes -x509 -subj ${SUBJECT_STRING} -keyout /server.key -out /server.crt >/dev/null 2>&1 + cat /server.key /server.crt > ${CERT_PEM} + rm -f /server.key /server.crt + fi + fi + sed -i -r -e 's/^# ssl-/ssl-/g' ${CONFIG_FILE} +fi + + +# Pre create database on the initiation of the container +if [ -n "${PRE_CREATE_DB}" ]; then + echo "=> About to create the following database: ${PRE_CREATE_DB}" + if [ -f "/data/.pre_db_created" ]; then + echo "=> Database had been created before, skipping ..." + else + echo "=> Starting InfluxDB ..." + exec /usr/bin/influxdb -config=${CONFIG_FILE} & + PASS=${INFLUXDB_INIT_PWD:-root} + arr=$(echo ${PRE_CREATE_DB} | tr ";" "\n") + + #wait for the startup of influxdb + RET=1 + while [[ RET -ne 0 ]]; do + echo "=> Waiting for confirmation of InfluxDB service startup ..." + sleep 3 + curl -k ${API_URL}/ping 2> /dev/null + RET=$? + done + echo "" + + for x in $arr + do + echo "=> Creating database: ${x}" + curl -s -k -X POST -d "{\"name\":\"${x}\"}" $(echo ${API_URL}'/db?u=root&p='${PASS}) + done + echo "" + + touch "/data/.pre_db_created" + fg + exit 0 + fi +else + echo "=> No database need to be pre-created" +fi + +echo "=> Starting InfluxDB ..." + +exec /usr/bin/influxdb -config=${CONFIG_FILE} diff --git a/README.md b/README.md index c5b44c8..24cd823 100644 --- a/README.md +++ b/README.md @@ -46,9 +46,10 @@ Open your browser to access `localhost:8083` to configure InfluxDB. Fill the por Alternatively, you can use RESTful API to talk to InfluxDB on port `8086`. For example, if you have problems with the initial database creation for version `0.9.x`, you can use the new `influx` cli tool to configure the database. While the container is running, you launch the tool with the following command: ``` - docker exec -ti influxdb-container-name /opt/influxdb/influx - Connected to http://localhost:8086 version 0.9.2.1 - InfluxDB shell 0.9.2.1 + docker exec -ti influxdb-container-name /usr/bin/influx + Visit https://enterprise.influxdata.com to register for updates, InfluxDB server management, and monitoring. + Connected to http://localhost:8086 version 0.9.6.1 + InfluxDB shell 0.9.6.1 > ```