前編の続き。
続きとはいってもGrowthforecastの複合グラフを作るスクリプト書いたっていうだけの記事です。
スクレイピングでがんばった!w
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
use strict; | |
use Web::Scraper; | |
use URI; | |
use LWP::UserAgent; | |
use YAML::Tiny; | |
use Data::Dumper; | |
use Getopt::Long; | |
my @graph_list; | |
my $config; | |
my $dry_run; | |
GetOptions('config=s' => \$config, 'dry-run' => \$dry_run); | |
die '--config is require.' unless $config; | |
# load config | |
my $yaml = YAML::Tiny->new; | |
$yaml = YAML::Tiny->read( $config ) or die "Can't open file $config."; | |
my $conf = $yaml->[0]; | |
sub scrape_links { | |
my $url = "http://$conf->{host}/"; | |
my $scraper = scraper { | |
process '//table[@class="zebra-striped"]/tr/td/a', 'link[]' => '@href'; | |
}; | |
my $result = $scraper->scrape(URI->new($url)); | |
my @paths; | |
for my $uri (@{$result->{link}}) { | |
my $path = $uri->path; | |
if($path =~ m|^/list/([^/]+)/([^/]+)|) { | |
my $service = $conf->{service_name} || ''; | |
my $section = $conf->{section_name} || ''; | |
if((!$service || ($service && $service eq $1)) | |
&& (!$section || ($section && $section eq $2)) ) { | |
push @paths, { | |
service_name => $1, | |
section_name => $2, | |
}; | |
} | |
} | |
} | |
#print Dumper \@paths; | |
return @paths; | |
} | |
sub scrape_graph_list { | |
my $url = "http://$conf->{host}/add_complex?service_name=hoge"; | |
my $scraper = scraper { | |
process '//select[@name="path-1"]/option', 'items[]' => scraper { | |
process '//*', 'key' => 'TEXT'; | |
process '//*', 'value' => '@value'; | |
}; | |
}; | |
my $result = $scraper->scrape(URI->new($url)); | |
return @{$result->{items}}; | |
} | |
sub filter_graph_list { | |
my ($path, $complex) = @_; | |
my @hit = (); | |
for my $row (@graph_list) { | |
my ($dummy, $service, $section, $graph) = split /\//, $row->{key}; | |
if($service eq $path->{service_name} && $section eq $path->{section_name}) { | |
if(!$complex || ($complex && $graph =~ $complex->{regex})) { | |
push @hit, $row; | |
} | |
} | |
} | |
return @hit; | |
} | |
sub request_add_complex { | |
my ($path, $complex, $graph_list) = @_; | |
my $config_graph_data = $conf->{graph_data} || {}; | |
my $sumup = $config_graph_data->{sumup} || '0'; | |
my $sort = $config_graph_data->{sort} || '19'; | |
my $type = $config_graph_data->{type} || 'AREA'; | |
my $gmode = $config_graph_data->{gmode} || 'gauge'; | |
my $stack = $config_graph_data->{stack} || '1'; | |
my $common = { | |
service_name => $path->{service_name}, | |
section_name => $path->{section_name}, | |
graph_name => $complex->{name}, | |
description => $complex->{description} || $complex->{name}, | |
sumup => $sumup, | |
sort => $sort, | |
}; | |
my $graph_data = { | |
'type-add' => $type, | |
'path-add' => '1', | |
'gmode-add' => $gmode, | |
'stack-add' => $stack, | |
}; | |
my @path_values = map {$_->{value}} @$graph_list; | |
# 1 | |
$graph_data->{'type-1'} = $type; | |
$graph_data->{'path-1'} = shift @path_values; | |
$graph_data->{'gmode-1'} = $gmode; | |
$graph_data->{'stack-1'} = $stack; | |
# 2 | |
$graph_data->{'type-2'} = []; | |
$graph_data->{'path-2'} = []; | |
$graph_data->{'gmode-2'} = []; | |
$graph_data->{'stack-2'} = []; | |
for my $path_value (@path_values) { | |
push @{$graph_data->{'type-2'}}, $type; | |
push @{$graph_data->{'path-2'}}, $path_value; | |
push @{$graph_data->{'gmode-2'}}, $gmode; | |
push @{$graph_data->{'stack-2'}}, $stack; | |
} | |
my $data = {%$common, %$graph_data}; | |
if($dry_run) { | |
printf("--------------------------------\n"); | |
printf("- service : %s\n",$path->{service_name}); | |
printf("- section : %s\n",$path->{section_name}); | |
printf("--------------------------------\n"); | |
printf("- add_complex : %s\n", $complex->{name}); | |
printf("--------------------------------\n"); | |
print Dumper $data; | |
} else { | |
printf("\n----------- /%s/%s/%s -----------\n", $path->{service_name}, $path->{section_name}, $complex->{name}); | |
post($data); | |
} | |
} | |
sub post { | |
my ($data) = @_; | |
my $ua = LWP::UserAgent->new; | |
$ua->timeout(3); | |
my $response = $ua->post("http://$conf->{host}/add_complex", $data); | |
if ($response->is_success) { | |
print $response->decoded_content; # or whatever | |
} else { | |
die $response->status_line; | |
} | |
} | |
sub main { | |
# grawthforecastのURL一覧を取得 | |
my @paths = scrape_links(); | |
die 'Not found paths.' if scalar @paths == 0; | |
# 設定されているメトリクス一覧を取得 | |
@graph_list = scrape_graph_list(); | |
die 'Not found graphlist.' if scalar @graph_list == 0; | |
for my $path (@paths) { | |
for my $cmplx (@{$conf->{complex}}) { | |
my @matches_graph_list = filter_graph_list($path, $cmplx); | |
next if scalar @matches_graph_list <= 1; | |
# 複合グラフURLへリクエスト | |
request_add_complex($path, $cmplx, \@matches_graph_list); | |
} | |
} | |
} | |
main(); |
書き捨てもいいところですね…
config.yaml
先ほどのdstatの複合グラフを作りたい時はこんなかんじでしょうか
--- host: localhost:5009 complex: - name: cpu_complex regex: ^cpu - name: disk_complex regex: ^dsk - name: io_complex regex: ^io - name: load_avalage_complex regex: ^la - name: mem_complex regex: ^mem - name: net_complex regex: ^net
スクリプト実行
以下のコマンドを叩くと自動で生成されます。
perl growthforecast_add_complex.pl --config=config.yaml
作る前にどのservice/sectionにどの複合グラフが生成されるか確認したい時には
dry-runオプションを付けると良いですね。
perl growthforecast_add_complex.pl --config=config.yaml --dry-run
configの補足
このスクリプト、ちょっとだけ汎用的に書いたので他のグラフでも使えます。少し補足。
--- # APIを叩くホスト host: localhost:5125 # service or sectionでのフィルタ。 # 指定しないとすべてのservice/sectionが対象となる service_name: hogehoge section_name: fugafuga # グラフデータのデフォルト値 graph_data: sumup: 1 # 合計値を表示するかどうか? sort: 19 # ソート順 type: AREA # 塗りつぶし AREA / LINE1 / LINE2 gmode: gauge # 実績:gauge / 差分:subtract # 複合グラフ設定 complex: - name: ping_complex # 複合グラフの名前 description: ping complex dayo # 複合グラフの説明(省略可) regex: ^ping. # 複合グラフをグルーピングするための正規表現。グラフ名でのフィルタになります - name: cpu_complex ...
Todo?
現状だと追加だけなので更新出来るようにするかも。
これでいろいろ夢広がりんぐになるとよいですね!