From fdc351da83de0fa0a9993c9b7227161f509d6cdc Mon Sep 17 00:00:00 2001 From: eson Date: Tue, 7 Jul 2020 16:17:05 +0800 Subject: [PATCH] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E6=95=B0=E6=8D=AE=E5=AD=98?= =?UTF-8?q?=E5=82=A8=E6=A0=BC=E5=BC=8F=E7=9A=84=E8=AE=BE=E8=AE=A1,=20?= =?UTF-8?q?=E7=94=A8ext=20=E5=AD=98=E5=82=A8=E5=AD=90=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E7=9A=84=E6=95=B0=E6=8D=AE=E5=86=85=E5=AE=B9.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 1 + go.sum | 2 ++ sql/intimate_extractor.sql | 24 +++++++++++++++++------- sql/intimate_source.sql | 4 ++++ store_test.go | 20 ++++++++++++++++++-- 5 files changed, 42 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 717c137..c38d4d3 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/474420502/hunter v0.1.2 github.com/go-sql-driver/mysql v1.5.0 github.com/go-yaml/yaml v2.1.0+incompatible // indirect + github.com/satori/go.uuid v1.2.0 github.com/tidwall/gjson v1.6.0 github.com/tidwall/pretty v1.0.1 // indirect gopkg.in/yaml.v2 v2.2.2 diff --git a/go.sum b/go.sum index baa05ba..1341f5c 100644 --- a/go.sum +++ b/go.sum @@ -56,6 +56,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= +github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/tebeka/selenium v0.9.9 h1:cNziB+etNgyH/7KlNI7RMC1ua5aH1+5wUlFQyzeMh+w= diff --git a/sql/intimate_extractor.sql b/sql/intimate_extractor.sql index 2f5011c..9364a70 100644 --- a/sql/intimate_extractor.sql +++ b/sql/intimate_extractor.sql @@ -2,13 +2,16 @@ create database if not exists `intimate_extractor`; use intimate_extractor; CREATE TABLE IF NOT EXISTS `anchor_info` ( - `platform` varchar(255) NOT NULL, - `anchor_name` varchar(255) NOT NULL, - `platform_url` text NOT NULL, - `channel` varchar(128) DEFAULT NULL, - `show_type` varchar(255) DEFAULT NULL, - `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`platform`, `anchor_name`), + `uid` varchar(36) NOT NULL, + `platform` varchar(255) NOT NULL, + `anchor_name` varchar(255) NOT NULL, + `platform_url` text NOT NULL, + `channel` varchar(128) DEFAULT NULL, + `show_type` varchar(255) DEFAULT NULL, + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`uid`), + KEY `platform_idx` (`platform`), + KEY `anchor_name_idx` (`anchor_name`), KEY `channel_idx` (`channel`), KEY `show_type_idx` (`show_type`), KEY `update_time_idx` (`update_time`) @@ -16,10 +19,13 @@ CREATE TABLE IF NOT EXISTS `anchor_info` ( ); CREATE TABLE IF NOT EXISTS `show_log` ( + `uid` varchar(36) NOT NULL, `platform` varchar(255) NOT NULL, `anchor_name` varchar(255) NOT NULL, `is_showing` tinyint(1) DEFAULT NULL, + `is_error` tinyint(1) DEFAULT NULL, + `followers` int(11) DEFAULT NULL, `views` int(11) DEFAULT NULL, `giver` json DEFAULT NULL, @@ -30,10 +36,14 @@ CREATE TABLE IF NOT EXISTS `show_log` ( `show_end_time` timestamp NULL DEFAULT NULL, `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, `ext` json DEFAULT NULL, + + `error` text DEFAULT NULL, + KEY `uid_idx` (`uid`), KEY `platform_idx` (`platform`), KEY `anchor_name_idx` (`anchor_name`), KEY `is_showing_idx` (`is_showing`), + KEY `is_error_idx` (`is_error`), KEY `followers_idx` (`followers`), KEY `views_idx` (`views`), KEY `gratuity_idx` (`gratuity`), diff --git a/sql/intimate_source.sql b/sql/intimate_source.sql index de03a13..3e3aadd 100644 --- a/sql/intimate_source.sql +++ b/sql/intimate_source.sql @@ -2,12 +2,16 @@ create database if not exists `intimate_source`; use intimate_source; CREATE TABLE IF NOT EXISTS `platform_openrec` ( + uid bigint AUTO_INCREMENT, `url` text NOT NULL, `target_type` varchar(64) NOT NULL, `source` longtext DEFAULT NULL, `ext` json DEFAULT NULL, + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, `operator` int DEFAULT 0, + `error_msg` text DEFAULT NULL, + PRIMARY KEY(`uid`), KEY `operator_idx` (`operator`), KEY `update_time_idx` (`update_time`), KEY `target_type_idx` (`target_type`) diff --git a/store_test.go b/store_test.go index ef09f41..80a9664 100644 --- a/store_test.go +++ b/store_test.go @@ -2,9 +2,25 @@ package main import ( "testing" + + "github.com/474420502/hunter" ) func TestStoreInsert(t *testing.T) { - // ht := hunter.NewHunter(openrecRanking) - // ht.Execute() + ht := hunter.NewHunter(openrecRanking) + ht.Execute() +} + +func TestStoreInsertCase1(t *testing.T) { + // extractorURI := "root:@tcp(127.0.0.1:4000)/intimate_extractor?charset=utf8" + // db, err := sql.Open("mysql", extractorURI) + // defer db.Close() + // if err != nil { + // t.Error(err) + // } + + // insertsql := "insert into anchor_info(uid, platform, anchor_name, platform_url) values(?,?,?,?)" + // _, err = db.Exec(insertsql, string(uuid.NewV4().String()), "1", "1", "1") + // t.Error(err, len(uuid.NewV4().String())) + }