更新数据存储格式的设计, 用ext 存储子任务的数据内容.

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/474420502/hunter v0.1.2
 	github.com/go-sql-driver/mysql v1.5.0
 	github.com/go-yaml/yaml v2.1.0+incompatible // indirect
+	github.com/satori/go.uuid v1.2.0
 	github.com/tidwall/gjson v1.6.0
 	github.com/tidwall/pretty v1.0.1 // indirect
 	gopkg.in/yaml.v2 v2.2.2

go.sum

@@ -56,6 +56,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc=
+github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
+github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/tebeka/selenium v0.9.9 h1:cNziB+etNgyH/7KlNI7RMC1ua5aH1+5wUlFQyzeMh+w=

sql/intimate_extractor.sql

@@ -2,13 +2,16 @@ create database if not exists `intimate_extractor`;
 use intimate_extractor;
 
 CREATE TABLE IF NOT EXISTS `anchor_info` (
-  `platform` varchar(255) NOT NULL,
-  `anchor_name` varchar(255) NOT NULL,
-  `platform_url` text NOT NULL,
-  `channel` varchar(128) DEFAULT NULL,
-  `show_type` varchar(255) DEFAULT NULL,
-  `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
-  PRIMARY KEY (`platform`, `anchor_name`),
+  `uid`           varchar(36)  NOT NULL,
+  `platform`      varchar(255) NOT NULL,
+  `anchor_name`   varchar(255) NOT NULL,
+  `platform_url`  text NOT NULL,
+  `channel`       varchar(128) DEFAULT NULL,
+  `show_type`     varchar(255) DEFAULT NULL,
+  `update_time`   timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+  PRIMARY KEY (`uid`),
+  KEY `platform_idx` (`platform`),
+  KEY `anchor_name_idx` (`anchor_name`),
   KEY `channel_idx` (`channel`),
   KEY `show_type_idx` (`show_type`),
   KEY `update_time_idx` (`update_time`)
@@ -16,10 +19,13 @@ CREATE TABLE IF NOT EXISTS `anchor_info` (
 );
 
 CREATE TABLE IF NOT EXISTS `show_log` (
+  `uid`      varchar(36)  NOT NULL,
   `platform` varchar(255) NOT NULL,
   `anchor_name` varchar(255) NOT NULL,
 
   `is_showing` tinyint(1) DEFAULT NULL,
+  `is_error` tinyint(1) DEFAULT NULL,
+
   `followers` int(11) DEFAULT NULL,
   `views` int(11) DEFAULT NULL,
   `giver` json DEFAULT NULL,
@@ -30,10 +36,14 @@ CREATE TABLE IF NOT EXISTS `show_log` (
   `show_end_time` timestamp NULL DEFAULT NULL,
   `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
   `ext` json DEFAULT NULL,
+  
+  `error` text DEFAULT NULL,
 
+  KEY `uid_idx` (`uid`),
   KEY `platform_idx` (`platform`),
   KEY `anchor_name_idx` (`anchor_name`),
   KEY `is_showing_idx` (`is_showing`),
+  KEY `is_error_idx` (`is_error`),
   KEY `followers_idx` (`followers`),
   KEY `views_idx` (`views`),
   KEY `gratuity_idx` (`gratuity`),

sql/intimate_source.sql

@@ -2,12 +2,16 @@ create database if not exists `intimate_source`;
 use intimate_source;
 
 CREATE TABLE IF NOT EXISTS `platform_openrec` (
+	uid bigint AUTO_INCREMENT,
 	`url` text NOT NULL,
 	`target_type` varchar(64) NOT NULL,
 	`source` longtext DEFAULT NULL,
 	`ext` json DEFAULT NULL,
+
 	`update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
 	`operator` int DEFAULT 0,
+	`error_msg` text DEFAULT NULL,
+	PRIMARY KEY(`uid`),
 	KEY `operator_idx` (`operator`),
 	KEY `update_time_idx` (`update_time`),
 	KEY `target_type_idx` (`target_type`)

store_test.go

@@ -2,9 +2,25 @@ package main
 
 import (
 	"testing"
+
+	"github.com/474420502/hunter"
 )
 
 func TestStoreInsert(t *testing.T) {
-	// ht := hunter.NewHunter(openrecRanking)
-	// ht.Execute()
+	ht := hunter.NewHunter(openrecRanking)
+	ht.Execute()
+}
+
+func TestStoreInsertCase1(t *testing.T) {
+	// extractorURI := "root:@tcp(127.0.0.1:4000)/intimate_extractor?charset=utf8"
+	// db, err := sql.Open("mysql", extractorURI)
+	// defer db.Close()
+	// if err != nil {
+	// 	t.Error(err)
+	// }
+
+	// insertsql := "insert into anchor_info(uid, platform, anchor_name, platform_url) values(?,?,?,?)"
+	// _, err = db.Exec(insertsql, string(uuid.NewV4().String()), "1", "1", "1")
+	// t.Error(err, len(uuid.NewV4().String()))
+
 }