eson/intimate_view
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

add charts

Browse Source
This commit is contained in:
eson 2020-08-20 19:29:22 +08:00
parent 17362cd7fe
commit c632837767
14 changed files with 517 additions and 194 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
goserver/count_tag.go Normal file
View File

@ -0,0 +1,65 @@
package main
import (
"encoding/json"
"fmt"
"log"
"time"
"github.com/gin-gonic/gin"
)
var tagCounter = make(map[string]*tagcounter)
type tagcounter struct {
Name string
LastTime time.Time
CountWord map[string]int
}
func init() {
}
func CountTag(cxt *gin.Context) {
platform := cxt.Query("platform")
var cw *tagcounter
if cw, ok := tagCounter[platform]; ok {
if time.Now().Sub(cw.LastTime).Minutes() <= 10 {
cxt.JSON(200, cw.CountWord)
return
}
}
sql := fmt.Sprintf(SqlTag, platform)
rows, err := StoreStreamer.Query(sql)
if err != nil {
cxt.Error(err)
return
}
cw = &tagcounter{}
cw.CountWord = make(map[string]int)
cw.Name = platform
cw.LastTime = time.Now()
tagCounter[platform] = cw
for rows.Next() {
var stag string
err = rows.Scan(&stag)
if err != nil {
log.Println(err)
}
var tag []string
json.Unmarshal([]byte(stag), &tag)
for _, t := range tag {
if _, ok := cw.CountWord[t]; ok {
cw.CountWord[t]++
} else {
cw.CountWord[t] = 1
}
}
}
cxt.JSON(200, cw.CountWord)
cw.LastTime = time.Now()
}

252
goserver/main.go
View File

@ -4,7 +4,6 @@ import (
"database/sql"
"encoding/json"
"fmt"
"log"
"net/http"
"strconv"
"strings"
@ -15,33 +14,6 @@ import (
)
var StoreStreamer *sql.DB
var SqlQuery string = `SELECT
ie.uid,
ie.platform,
ie.user_id,
ie.user_name ,
ie.live_url ,
ie.tags ,
cl.followers ,
cl.views ,
cl.gratuity ,
cl.live_title,
cl.live_start_time ,
cl.live_end_time ,
cl.update_time
From
(
SELECT
*
FROM
intimate_extractor.streamer
WHERE
platform = "twitcasting"
AND operator = 0
AND latest_log_uid is not NULL limit %s,%s) ie
JOIN intimate_extractor.collect_log cl
WHERE
ie.latest_log_uid = cl.log_uid; `
func init() {
db, err := sql.Open("mysql", InitConfig.Database.ExtractorURI)
@ -51,121 +23,12 @@ func init() {
StoreStreamer = db
}
// ie.uid,
// ie.platform,
// ie.user_id,
// ie.user_name ,
// ie.live_url ,
// ie.tags ,
// cl.followers ,
// cl.views ,
// cl.gratuity ,
// cl.live_title,
// cl.live_start_time ,
// cl.live_end_time ,
// cl.update_time
type ObjectTwistcasting struct {
Uid int64
Platform string
UserId string
UserName string
LiveUrl string
Tags []string
Followers int64
Views int64
Gratuity int64
LiveTitle string
LiveStartTime *time.Time
LiveEndTime *time.Time
UpdateTime *time.Time
}
type Result struct {
Code int
Error string
Data interface{}
}
func TwitcastingQuery(cxt *gin.Context) {
var err error
page, err := strconv.Atoi(cxt.Query("page"))
if err != nil {
cxt.Error(err)
}
psize, err := strconv.Atoi(cxt.Query("psize"))
if err != nil {
cxt.Error(err)
}
if psize > 100 {
cxt.Error(fmt.Errorf("page size <= 100"))
}
start := (page - 1) * 200
// end := start + 200
ssql := fmt.Sprintf(SqlQuery, strconv.Itoa(start), strconv.Itoa(psize))
rows, err := StoreStreamer.Query(ssql)
if err != nil {
cxt.Error(err)
}
var ots []*ObjectTwistcasting
for rows.Next() {
ot := &ObjectTwistcasting{}
var lstm, letm, utm sql.NullTime
var tags string
err = rows.Scan(
&ot.Uid,
&ot.Platform,
&ot.UserId,
&ot.UserName,
&ot.LiveUrl,
&tags,
&ot.Followers,
&ot.Views,
&ot.Gratuity,
&ot.LiveTitle,
&lstm,
&letm,
&utm,
)
if err != nil {
cxt.Error(err)
return
}
if !lstm.Valid {
ot.LiveStartTime = nil
} else {
ot.LiveStartTime = &lstm.Time
}
if !letm.Valid {
ot.LiveEndTime = nil
} else {
ot.LiveEndTime = &letm.Time
}
if !utm.Valid {
ot.UpdateTime = nil
} else {
ot.UpdateTime = &utm.Time
}
if err = json.Unmarshal([]byte(tags), &ot.Tags); err != nil {
log.Println(tags)
}
ots = append(ots, ot)
}
r := &Result{Code: 200}
r.Data = ots
log.Println(len(ots))
if retdata, err := json.Marshal(r); err != nil {
cxt.Error(err)
} else {
cxt.JSON(r.Code, string(retdata))
}
}
func Cors() gin.HandlerFunc {
return func(c *gin.Context) {
method := c.Request.Method //请求方法
@ -202,10 +65,125 @@ func Cors() gin.HandlerFunc {
}
}
type ObjectQuery struct {
Uid int64
Platform string
UserId string
UserName string
LiveUrl string
Tags []string
Followers int64
Views int64
Gratuity int64
LiveTitle string
LiveStartTime *time.Time
LiveEndTime *time.Time
UpdateTime *time.Time
}
func Query(cxt *gin.Context, platform string) {
var err error
page, err := strconv.Atoi(cxt.Query("page"))
if err != nil {
cxt.Error(err)
return
}
psize, err := strconv.Atoi(cxt.Query("psize"))
if err != nil {
cxt.Error(err)
return
}
if psize > 100 {
cxt.Error(fmt.Errorf("page size <= 100"))
return
}
start := (page - 1) * psize
// end := start + 200
ssql := fmt.Sprintf(SqlQuery, platform, strconv.Itoa(start), strconv.Itoa(psize))
rows, err := StoreStreamer.Query(ssql)
if err != nil {
cxt.Error(err)
return
}
var ots []*ObjectQuery
for rows.Next() {
ot := &ObjectQuery{}
var view, gratuity sql.NullInt64
var lstm, letm, utm sql.NullTime
var tags, livetitle sql.NullString
err = rows.Scan(
&ot.Uid,
&ot.Platform,
&ot.UserId,
&ot.UserName,
&ot.LiveUrl,
&tags,
&ot.Followers,
&view,
&gratuity,
&livetitle,
&lstm,
&letm,
&utm,
)
if err != nil {
cxt.Error(err)
return
}
if !lstm.Valid {
ot.LiveStartTime = nil
} else {
ot.LiveStartTime = &lstm.Time
}
if !letm.Valid {
ot.LiveEndTime = nil
} else {
ot.LiveEndTime = &letm.Time
}
if !utm.Valid {
ot.UpdateTime = nil
} else {
ot.UpdateTime = &utm.Time
}
if livetitle.Valid {
ot.LiveTitle = livetitle.String
}
if view.Valid {
ot.Views = view.Int64
}
if gratuity.Valid {
ot.Gratuity = gratuity.Int64
}
if err = json.Unmarshal([]byte(tags.String), &ot.Tags); err != nil {
// log.Println(tags)
}
ots = append(ots, ot)
}
r := &Result{Code: 200}
r.Data = ots
// log.Println(len(ots))
if retdata, err := json.Marshal(r); err != nil {
cxt.Error(err)
} else {
cxt.JSON(r.Code, string(retdata))
}
}
func main() {
engine := gin.New() //r := gin.Default() //使用默认中间件
engine.Use(gin.Logger())
engine.Use(Cors())
engine.GET("twitcasting/query", TwitcastingQuery)
engine.GET("openrec/query", OpenrecQuery)
engine.GET("twitch/query", TwitchQuery)
engine.GET("tag/count", CountTag)
engine.Run(":5500")
}

8
goserver/main_test.go
View File

@ -1,7 +1,13 @@
package main
import "testing"
import (
"testing"
)
func TestMain(t *testing.T) {
main()
}
func TestCountTag(t *testing.T) {
}

7
goserver/openrec.go Normal file
View File

@ -0,0 +1,7 @@
package main
import "github.com/gin-gonic/gin"
func OpenrecQuery(cxt *gin.Context) {
Query(cxt, "openrec")
}

9
goserver/twitcasting.go Normal file
View File

@ -0,0 +1,9 @@
package main
import (
"github.com/gin-gonic/gin"
)
func TwitcastingQuery(cxt *gin.Context) {
Query(cxt, "twitcasting")
}

8
goserver/twitch.go Normal file
View File

@ -0,0 +1,8 @@
package main
import "github.com/gin-gonic/gin"
// TwitchQuery
func TwitchQuery(cxt *gin.Context) {
Query(cxt, "twitch")
}

34
goserver/var_sql.go Normal file
View File

@ -0,0 +1,34 @@
package main
// SqlTag 获取 tag
var SqlTag string = `SELECT tags FROM intimate_extractor.streamer
WHERE platform = "%s" AND tags IS NOT NULL`
//SqlQuery 获取 网站数据
var SqlQuery string = `SELECT
ie.uid,
ie.platform,
ie.user_id,
ie.user_name ,
ie.live_url ,
ie.tags ,
cl.followers ,
cl.views ,
cl.gratuity ,
cl.live_title,
cl.live_start_time ,
cl.live_end_time ,
cl.update_time
From
(
SELECT
*
FROM
intimate_extractor.streamer
WHERE
platform = "%s"
AND operator = 0
AND latest_log_uid is not NULL limit %s,%s) ie
JOIN intimate_extractor.collect_log cl
WHERE
ie.latest_log_uid = cl.log_uid; `

34
package-lock.json generated
View File

@ -5066,6 +5066,30 @@
"safer-buffer": "^2.1.0"
}
},
"echarts": {
"version": "4.8.0",
"resolved": "https://registry.npmjs.org/echarts/-/echarts-4.8.0.tgz",
"integrity": "sha512-YwShpug8fWngj/RlgxDaYrLBoD+LsZUArrusjNPHpAF+is+gGe38xx4W848AwWMGoi745t3OXM52JedNrv+F6g==",
"requires": {
"zrender": "4.3.1"
}
},
"echarts-for-react": {
"version": "2.0.16",
"resolved": "https://registry.npmjs.org/echarts-for-react/-/echarts-for-react-2.0.16.tgz",
"integrity": "sha512-VmHCktay2qKt/+wpL/C7thbvIa7dYBEey0/U4Zaqo+qeA4wx+uiCd5NeCsPIhD/0Pv+2qqNswqiNiUCtcgccOw==",
"requires": {
"fast-deep-equal": "^2.0.1",
"size-sensor": "^1.0.0"
},
"dependencies": {
"fast-deep-equal": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz",
"integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk="
}
}
},
"ee-first": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
@ -14024,6 +14048,11 @@
"resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz",
"integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg=="
},
"size-sensor": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/size-sensor/-/size-sensor-1.0.1.tgz",
"integrity": "sha512-QTy7MnuugCFXIedXRpUSk9gUnyNiaxIdxGfUjr8xxXOqIB3QvBUYP9+b51oCg2C4dnhaeNk/h57TxjbvoJrJUA=="
},
"slash": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/slash/-/slash-2.0.0.tgz",
@ -16392,6 +16421,11 @@
"camelcase": "^5.0.0",
"decamelize": "^1.2.0"
}
},
"zrender": {
"version": "4.3.1",
"resolved": "https://registry.npmjs.org/zrender/-/zrender-4.3.1.tgz",
"integrity": "sha512-CeH2TpJeCdG0TAGYoPSAcFX2ogdug1K7LIn9UO/q9HWqQ54gWhrMAlDP9AwWYMUDhrPe4VeazQ4DW3msD96nUQ=="
}
}
}

2
package.json
View File

@ -10,6 +10,8 @@
"@testing-library/user-event": "^11.1.0",
"antd": "^4.5.4",
"craco-less": "^1.17.0",
"echarts": "^4.8.0",
"echarts-for-react": "^2.0.16",
"react": "^16.13.1",
"react-dom": "^16.13.1",
"react-scripts": "^3.4.3",

19
src/App.js
View File

@ -5,9 +5,12 @@ import Icon, {
import { Button, Layout, Menu } from 'antd';
import 'antd/dist/antd.css';
import React from 'react';
import ReactDom from 'react-dom';
import './App.less';
import DataTable from './Table';
import ContentTable from './ContentTable';
import DataTableTest from './TableTest';
import CountCharts from './Graph';
const { Header, Content, Footer, Sider } = Layout;
@ -38,10 +41,15 @@ class App extends React.Component {
</Button>
<Menu theme="dark" mode="inline" defaultSelectedKeys={['1']}
defaultOpenKeys={['sub1']}
>
<Menu.Item key="1" icon={<DatabaseFilled/>} >
数据展示
<Menu.Item key="openrec" icon={<DatabaseFilled/>} onClick={(e)=>{ this.refs.ctable.changePlatform( e.key) ; }} >
openrec
</Menu.Item>
<Menu.Item key="twitcasting" icon={<DatabaseFilled/>} onClick={(e)=>{ this.refs.ctable.changePlatform( e.key) ; }} >
twitcasting
</Menu.Item>
<Menu.Item key="twitch" icon={<DatabaseFilled/>} onClick={(e)=>{ this.refs.ctable.changePlatform( e.key) ; }} >
twitch
</Menu.Item>
</Menu>
</Sider>
@ -53,9 +61,8 @@ class App extends React.Component {
</span>
</Header>
<Content style={{ margin: '0 8px' }}>
<div style={{ padding: 24, background: '#fff', minHeight: 780 }}>
<DataTable ></DataTable >
</div>
{/* <ContentTable ref="ctable" platform={"openrec"}></ContentTable> */}
<CountCharts></CountCharts>
</Content>
<Footer style={{ textAlign: 'center' }}>
Intimate ©2020 Created by eson

21
src/ContentTable.js Normal file
View File

@ -0,0 +1,21 @@
import React from 'react';
import DataTable from './Table.js';
class ContentTable extends React.Component {
changePlatform(p) {
// this.refs.table.firstPage();
// this.setState({platform: p});
// this.refs.table.setState({platform: p})
this.refs.table.updatePlatform(p);
}
render() {
// const {platform} = this.state;
return (<div style={{ padding: 24, background: '#fff', minHeight: 780 }}>
<DataTable ref="table" ></DataTable >
</div>)
}
}
export default ContentTable;

151
src/Graph.js Normal file
View File

@ -0,0 +1,151 @@
import React from 'react';
import ReactEcharts from 'echarts-for-react';
import reqwest from 'reqwest';
// function genData(count) {
// var nameList = [
// '赵', '钱', '孙', '李', '周', '吴', '郑', '王', '冯', '陈', '褚', '卫', '蒋', '沈', '韩', '杨', '朱', '秦', '尤', '许', '何', '吕', '施', '张', '孔', '曹', '严', '华', '金', '魏', '陶', '姜', '戚', '谢', '邹', '喻', '柏', '水', '窦', '章', '云', '苏', '潘', '葛', '奚', '范', '彭', '郎', '鲁', '韦', '昌', '马', '苗', '凤', '花', '方', '俞', '任', '袁', '柳', '酆', '鲍', '史', '唐', '费', '廉', '岑', '薛', '雷', '贺', '倪', '汤', '滕', '殷', '罗', '毕', '郝', '邬', '安', '常', '乐', '于', '时', '傅', '皮', '卞', '齐', '康', '伍', '余', '元', '卜', '顾', '孟', '平', '黄', '和', '穆', '萧', '尹', '姚', '邵', '湛', '汪', '祁', '毛', '禹', '狄', '米', '贝', '明', '臧', '计', '伏', '成', '戴', '谈', '宋', '茅', '庞', '熊', '纪', '舒', '屈', '项', '祝', '董', '梁', '杜', '阮', '蓝', '闵', '席', '季', '麻', '强', '贾', '路', '娄', '危'
// ];
// var legendData = [];
// var seriesData = [];
// var selected = {};
// for (var i = 0; i < count; i++) {
// var name = Math.random() > 0.65
// ? makeWord(4, 1) + '·' + makeWord(3, 0)
// : makeWord(2, 1);
// legendData.push(name);
// seriesData.push({
// name: name,
// value: Math.round(Math.random() * 100000)
// });
// selected[name] = i < 6;
// }
// return {
// legendData: legendData,
// seriesData: seriesData,
// selected: selected
// };
// function makeWord(max, min) {
// var nameLen = Math.ceil(Math.random() * max + min);
// var name = [];
// for (var i = 0; i < nameLen; i++) {
// name.push(nameList[Math.round(Math.random() * nameList.length - 1)]);
// }
// return name.join('');
// }
// }
function parseData(cw={}) {
var legendData = [];
var seriesData = [];
var selected = {};
for(var name in cw){
legendData.push(name);
seriesData.push({
name: name,
value: cw[name]
})
selected[name] = false
}
return { legendData: legendData, seriesData: seriesData, selected:selected }
}
const getRandomuserParams = params => {
return {
platform: params.platform,
// ...params,
};
};
class CountCharts extends React.Component {
state = {
platform: "twitch",
data: {}
}
fetch = (params = {}) => {
// console.log(platform);
reqwest({
url: 'http://localhost:5500/tag/count',
method: 'get',
type: 'json',
data: getRandomuserParams(params),
}).then(data => {
var result = JSON.parse(data);
console.log("result", result);
this.setState({
data: parseData(result),
});
});
};
componentDidMount() {
this.fetch(this.state);
}
render() {
// this.fetch(this.state);
const {data} = this.state;
const option = {
title: {
text: '同名数量统计',
subtext: '纯属虚构',
left: 'center'
},
tooltip: {
trigger: 'item',
formatter: '{a} <br/>{b} : {c} ({d}%)'
},
legend: {
type: 'scroll',
orient: 'vertical',
right: 10,
top: 20,
bottom: 20,
data: data.legendData,
selected: data.selected
},
series: [
{
name: '标签',
type: 'pie',
radius: '55%',
center: ['40%', '50%'],
data: data.seriesData,
emphasis: {
itemStyle: {
shadowBlur: 10,
shadowOffsetX: 0,
shadowColor: 'rgba(0, 0, 0, 0.5)'
}
}
}
],
};
return ( <div style={{ padding: 24, background: '#fff', minHeight: 780 }} >
<ReactEcharts
option={option}
style={{height: '350px', width: '1000px'}}
className='react_for_echarts' />
</div>
);
}
};
export default CountCharts;

97
src/Table.js
View File

@ -1,47 +1,40 @@
import React, { useState } from 'react';
import { Table, Input, InputNumber, Popconfirm, Form, Tag } from 'antd';
import reqwest from 'reqwest';
import './Table.less';
import './Table.less';
import { useForm } from 'antd/lib/form/Form';
const columns = [
// {
// title: 'uid',
// dataIndex: 'Uid',
// editable: false,
// width: "7%",
// },
{
title: '平台',
dataIndex: 'Platform',
editable: false,
key: 'Platform',
// width: "8%",
},
{
title: 'userid',
dataIndex: 'UserId',
editable: false,
key: 'UserId',
// width: "7%",
},
{
title: '名称',
dataIndex: 'UserName',
editable: false,
key: 'UserName',
// width: "7%",
},
// {
// title: '直播地址',
// dataIndex: 'LiveUrl',
// editable: false,
// width: "5%",
// },
{
title: '标签',
dataIndex: 'Tags',
editable: false,
render: tags => (
<>
{tags.map( tag => {
{
tags != null ?
tags.map( tag => {
let color = "purple";
if (tag.length < 3) {
color = 'green';
@ -56,42 +49,27 @@ const columns = [
{tag}
</Tag>
);
})}
}
) : null}
</> )
},
{
title: '粉丝数(关注)',
dataIndex: 'Followers',
key: 'Followers',
width: "8%",
},
{
title: '礼物数(币|钱)',
dataIndex: 'Gratuity',
key: 'Gratuity',
width: "8%",
},
// {
// title: '直播地址',
// dataIndex: 'LiveUrl',
// width: "5%",
// },
// {
// title: '直播标题',
// dataIndex: 'LiveTitle',
// },
// {
// title: '近直播开始时间',
// dataIndex: 'LiveStartTime',
// width: "7%",
// },
// {
// title: '近直播结束时间',
// dataIndex: 'LiveEndTime',
// width: "7%",
// },
{
title: '数据更新时间',
dataIndex: 'UpdateTime',
key: 'UpdateTime',
// width: "7%",
},
];
@ -112,8 +90,11 @@ const columns = [
class DataTable extends React.Component {
state = {
data: [],
platform: "openrec",
pagination: {
current: 1,
pageSize: 20,
@ -128,35 +109,53 @@ class DataTable extends React.Component {
{
title: 'uid',
dataIndex: 'Uid',
key: 'Uid',
},
{
title: '直播地址',
dataIndex: 'LiveUrl',
key: 'LiveUrl',
},
{
title: '直播标题',
dataIndex: 'LiveTitle',
key: 'LiveTitle',
},
{
title: '近直播开始时间',
dataIndex: 'LiveStartTime',
key: 'LiveStartTime',
},
{
title: '近直播结束时间',
dataIndex: 'LiveEndTime',
key: 'LiveEndTime',
},
];
var data = [record];
return <Table bordered={true} type="small" columns={ecolumns} dataSource={data} pagination={false} />;
return <Table rowClassName="subtable" size="small" bordered={true} columns={ecolumns} dataSource={data} pagination={false} />;
};
updatePlatform(p) {
const { pagination } = this.state;
pagination.current = 1;
this.setState({platform:p}, ()=>{
this.fetch({
pagination
});
});
}
componentDidMount() {
const { pagination } = this.state;
this.fetch({
pagination
});
const {platform} = this.state;
this.updatePlatform(platform);
}
handleTableChange = (pagination, filters, sorter) => {
console.log(filters, sorter);
// console.log(filters, sorter);
this.fetch({
sortField: sorter.field,
sortOrder: sorter.order,
@ -167,8 +166,10 @@ class DataTable extends React.Component {
fetch = (params = {}) => {
this.setState({ loading: true });
const {platform} = this.state;
console.log(platform);
reqwest({
url: 'http://localhost:5500/twitcasting/query',
url: 'http://localhost:5500/' + platform + '/query',
method: 'get',
type: 'json',
data: getRandomuserParams(params),
@ -186,16 +187,12 @@ class DataTable extends React.Component {
};
render() {
const { data, pagination, loading } = this.state;
return (
return (
<Table
bordered={true}
size={"middle"}
// scroll={{ x: "150vh", y:"800" }}
columns={columns}
dataSource={data}

4
src/Table.less
View File

@ -5,4 +5,8 @@
position: absolute;
top: 100%;
font-size: 12px;
}
.subtable {
background: rgb(220, 233, 231);
}