InfluxDB

Retention policy

Periodinis duomenu salinimas kas savaite:

create retention policy removal on parser duration 1w replication 1;

Arba kuriant db, iskart nurodyti

CREATE DATABASE "parser" WITH DURATION 1h REPLICATION 1 SHARD DURATION 24h NAME "parser"

db crashed

rm -rf /var/lib/influxdb/data/
pkill -9 influx
/etc/init.d/influxdb start

Perl RAW

TODO:

• laiko formatas pasimetes
• Reiktų, kad iš paduodamų parametrų išvalytume white space, space ir , juos escapindami su \
• Naudoti InfluxDB::LineProtocol :D

#!/usr/bin/per
use AnyEvent;
use AnyEvent::Socket;
use AnyEvent::Handle;
use AnyEvent::InfluxDB;
use DateTime;
use DateTime::Precise;
use strict;
use warnings;

my $db = AnyEvent::InfluxDB->new(
    server => 'http://127.0.0.1:8086',
    username => 'admin',
    password => 'admin',
 on_request => sub {
        my ($method, $url, $post_data) = @_;
        print "$method $url\n";
        print "$post_data\n" if $post_data;
    }
);

my $data = { 'deployment' => 'app2', 'email' => 'justinas@eofnet.lt', 'status' => '"sent"', 'status_msg' => '"Message queued"' };
# split domain data
my @splitas = split /@/, $data->{email};
$data->{domain} = $splitas[1];
my $cv = AE::cv;
$db->write(
database => 'parser',
data => [
{
            measurement => 'log_entries',
            tags => {
                deployment => "$data->{deployment}",
                email => "$data->{email}",
                domain => "$data->{domain}",
            },
            fields => {
                status => $data->{status},
                status_msg => $data->{status_msg},
            },
            time => time()
        }
    ],
on_success => $cv,
    on_error => sub {
        $cv->croak("Failed to write data: @_");
    } 
);
$cv->recv;

Perl easy way

Nežinau kas darė visą influxdb api perl'ui bet jis visiškai netinkamas naudojimui. Threadintas scriptas neatlaikė net 5k užklausų per minutę ir padarė segmentation fault crash'ą.

kernel: [5591153.130405] write_flux.pl[4515]: segfault at 78 ip 00007fa7102f829b sp 00007fa6fa7fba20 error 4 in EV.so[7fa7102dc000+24000]

Todėl nieko kito neliko tik naudoti external curl įrankį, dummy thread bloke. Negaudysim jokių error kodų, tiesiog forkinsim procesus ir stdoutą redirektinsim į /dev/null ;-) Aišku galima nenaudoti net influxDB::LineProtocol, bet patingėjau, tiesiog tokiu atveju reiktų visus space ir kablelius escapinti slashu \. Laiko formatas irgi UTC, todėl frontende kažkur, kur norima atvaizduoti informaciją reiks pridėt pora valandų, norint, kad rodytų Lietuvos laiku.

#!/usr/bin/perl
use InfluxDB::LineProtocol qw(data2line);
use strict;
use warnings;
use Thread;
my $measurement = 'log_entries';
my $influx_db = 'parser';
my $influx_user = 'admin';
my $influx_pass = 'admin';

# reimplemented version, not using any of influxdb API's, only plain external curl command
sub fluxdata2 {
my ($data) = @_;
my @splitas = split /@/, $data->{email};
$data->{domain} = $splitas[1];
my $timestamp = `date +%s%N`;
my $tags = { deployment => $data->{deployment}, email => $data->{email}, domain => $data->{domain} };
my $fields = { status => $data->{status}, status_msg => $data->{status_msg} };
my $influx_line = data2line($measurement, $fields, $tags,$timestamp);
my $curl = "curl -i -XPOST 'http://localhost:8086/write?db=".$influx_db."&u=".$influx_user."&p=".$influx_pass."' --data-binary '$influx_line' > /dev/null 2>&1&";
system($curl);
}
my $data = { 'deployment' => 'app2', 'email' => 'justinas@eofnet.lt', 'status' => 'sent', 'status_msg' => 'Message queued' };
for my $i (1 .. 5000) {
    threads->create(\&fluxdata2, $data);
}
print "Script is done!\n";
print "Waiting for threads to finish...\n";
map { $_->join(); } threads->list();

3'čia versija

#!/usr/bin/perl -w
my $measurement = 'log_entries';
my $influx_db = 'parser';
my $influx_user = 'admin';
my $influx_pass = 'admin';

use strict;
use warnings;
use Mojo::UserAgent;
use InfluxDB::LineProtocol qw(data2line);
my $ua  = Mojo::UserAgent->new;
Mojo::IOLoop->start unless Mojo::IOLoop->is_running;

# reimplemented version 3, not using any of influxdb API's nor external commands, just plain post to it's API
sub fluxdata {
my ($data) = @_;
my @splitas = split /@/, $data->{email};
$data->{domain} = $splitas[1];
my $tags = { deployment => $data->{deployment}, instance => $data->{instance}, campaign => $data->{campaign}, email => $data->{email}, domain => $data->{domain} };
my $fields = { status => $data->{status}, status_msg => $data->{status_msg} };
my $influx_line = data2line($measurement, $fields, $tags);
$ua->post('http://localhost:8086/write?db='.$influx_db.'&u='.$influx_user.'&p='.$influx_pass, {'Content-Type' => 'text/plain'}, $influx_line);
}

my $data = { 'deployment' => 'testas', 'campaign' => '1', 'instance' => '111.111.111.111', 'email' => 'justinas@eofnet.lt', 'status' => 'sent', 'status_msg' => 'Message queued' };
fluxdata($data);