diff --git a/demo/Blog/misc/mysql/init-db.sql b/demo/Blog/misc/mysql/init-db.sql index dc75a8b..3a98351 100644 --- a/demo/Blog/misc/mysql/init-db.sql +++ b/demo/Blog/misc/mysql/init-db.sql @@ -17,7 +17,7 @@ create table comments ( sender varchar(64) not null, email varchar(64) not null, url varchar(1024), - body text, + body text not null, created timestamp(0) default now() not null, post bigint not null, foreign key (post) references posts(id) diff --git a/util/import-model.pl b/util/import-model.pl new file mode 100755 index 0000000..7815e18 --- /dev/null +++ b/util/import-model.pl @@ -0,0 +1,127 @@ +#!/usr/bin/env perl + +use strict; +use warnings; + +use Getopt::Long; +use lib 'lib'; +use Params::Util qw( _HASH ); +#use JSON::XS (); +use JSON::Syck; +#use WWW::OpenResty::Simple; +use Data::Dumper; +use Data::Structure::Util qw(_utf8_on _utf8_off); +use DBI; + +sub usage { + my $progname; + if ($0 =~ m{[^/\\]+$}) { + $progname = $&; + } + return <<_EOC_; +Usage: $progname [options] +Options: + --user OpenResty user (i.e. agentzh.Admin) + --password OpenResty password for the user specified + --model OpenResty model name being imported + --server OpenResty server hostname + --step Size of the bulk insertion group + --retries Number of automatic retries when failures occur + --no-id Skipped the id field in the records being imported + --skip N Skipped the first N rows in the input file (default 0) + --add-id N Add ID started from N (default 1) + --update-col Update records according to the column given + --ignore-dup-error Ignore dupliciate errors +_EOC_ +} + +my $server = 'api.openresty.org'; +my $step = 20; +my $skip = 0; +my $add_id = 1; +my $update_col; +GetOptions( + 'help|h' => \(my $help), + 'user|u=s' => \(my $user), + 'model=s' => \(my $model), + 'server=s' => \$server, + 'password=s' => \(my $password), + 'step=i' => \$step, + 'reset' => \(my $reset), + 'retries=i' => \(my $retries), + 'no-id' => \(my $no_id), + 'skip=i' => \$skip, + 'add-id=i' => \$add_id, + 'ignore-dup-error' => \(my $ignore_dup_error), + 'update-col=s' => \$update_col, + 'port=s' => \(my $port), +) or die usage(); + +if ($help) { print usage() } + +$user or die "No --user given.\n"; +$model or die "No --model given.\n"; + +# my $json_xs = JSON::XS->new->utf8->allow_nonref; + +my $database = 'test'; + +my $dsn = "DBI:mysql:database=$database;host=$server;port=$port"; +my $dbh = DBI->connect($dsn, $user, $password, { RaiseError => 1, AutoCommit => 0 }); + +if ($reset) { + $dbh->do("truncate $model"); +} + +if ($update_col) { $step = 1 } + +my @rows; +my $inserted = 0; +local $| = 1; + +my ($sth); + +while (<>) { + #select(undef, undef, undef, 0.1); + #warn "count: ", scalar(@elems), "\n"; + next if $. <= $skip; + # my $row = $json_xs->decode($_); + my $row = JSON::Syck::Load($_); + + + if (!defined $row->{id}) { + #warn "ADDing id...\n"; + $row->{id} = $add_id++; + } + if ($no_id) { + delete $row->{id}; + } + + if (!defined $sth) { + my @cols = sort keys %$row; + my $cols = join ',', @cols; + my @holders = map { '?' } @cols; + my $holders = join ',', @holders; + $sth = $dbh->prepare("insert into ($cols) values ($holders)"); + } + + my @vals; + for my $col (sort keys %$row) { + push @vals, $row->{$col}; + } + + $sth->execute(@vals); + $inserted++; + + if (@rows % $step == 0) { + $inserted += insert_rows(\@rows); + @rows = (); + print STDERR "\r", ($update_col ? "Updated" : "Inserted"), " rows: $inserted (row $.)"; + } +} + +print STDERR "\n$inserted row(s) inserted.\n"; + +warn "\nFor tatal $inserted records inserted.\n"; +#print encode('UTF-8', YAML::Syck::Dump(\@rows)); +