openresty/util/import-model.pl

131 lines
3.2 KiB
Perl
Executable File

#!/usr/bin/env perl
use strict;
use warnings;
use Getopt::Long;
use lib 'lib';
use Params::Util qw( _HASH );
use JSON::XS ();
use JSON::Syck;
#use WWW::OpenResty::Simple;
use Data::Dumper;
use Data::Structure::Util qw(_utf8_on _utf8_off);
use DBI;
sub usage {
my $progname;
if ($0 =~ m{[^/\\]+$}) {
$progname = $&;
}
return <<_EOC_;
Usage: $progname [options] <json_file>
Options:
--user <user> OpenResty user (i.e. agentzh.Admin)
--password <s> OpenResty password for the user specified
--model <model> OpenResty model name being imported
--server <host> OpenResty server hostname
--step <num> Size of the bulk insertion group
--retries <num> Number of automatic retries when failures occur
--no-id Skipped the id field in the records being imported
--skip N Skipped the first N rows in the input file (default 0)
--add-id N Add ID started from N (default 1)
--update-col <col> Update records according to the column given
--ignore-dup-error Ignore dupliciate errors
_EOC_
}
my $server = 'api.openresty.org';
my $step = 20;
my $skip = 0;
my $add_id = 1;
my $update_col;
GetOptions(
'help|h' => \(my $help),
'user|u=s' => \(my $user),
'model=s' => \(my $model),
'server=s' => \$server,
'password=s' => \(my $password),
'step=i' => \$step,
'reset' => \(my $reset),
'retries=i' => \(my $retries),
'no-id' => \(my $no_id),
'skip=i' => \$skip,
'add-id=i' => \$add_id,
'ignore-dup-error' => \(my $ignore_dup_error),
'update-col=s' => \$update_col,
'port=s' => \(my $port),
) or die usage();
if ($help) { print usage() }
$user or die "No --user given.\n";
$model or die "No --model given.\n";
my $json_xs = JSON::XS->new->utf8->allow_nonref;
my $database = 'test';
$port ||= 3306;
my $dsn = "DBI:mysql:database=$database;host=$server;port=$port;mysql_server_prepare=1";
my $dbh = DBI->connect($dsn, $user, $password, { RaiseError => 1, AutoCommit => 0 });
if ($reset) {
$dbh->do("truncate $model");
}
if ($update_col) { $step = 1 }
my @rows;
my $inserted = 0;
local $| = 1;
my ($sth);
my @cols;
while (<>) {
#select(undef, undef, undef, 0.1);
#warn "count: ", scalar(@elems), "\n";
next if $. <= $skip;
my $row = $json_xs->decode($_);
#my $row = JSON::Syck::Load($_);
if (!defined $row->{id}) {
#warn "ADDing id...\n";
$row->{id} = $add_id++;
}
if ($no_id) {
delete $row->{id};
}
if (!defined $sth) {
@cols = reverse sort keys %$row;
my $cols = join ',', @cols;
my @holders = map { '?' } @cols;
my $holders = join ',', @holders;
$sth = $dbh->prepare("insert into $model ($cols) values ($holders)");
}
my @vals;
for my $col (@cols) {
push @vals, $row->{$col};
}
$sth->execute(@vals);
$inserted++;
if ($inserted % $step == 0) {
print STDERR "\r", ($update_col ? "Updated" : "Inserted"), " rows: $inserted (row $.)";
}
}
print STDERR "\n$inserted row(s) inserted.\n";
$dbh->disconnect();
warn "\nFor tatal $inserted records inserted.\n";
#print encode('UTF-8', YAML::Syck::Dump(\@rows));