add: duplicate tags

add: field sorted
fix: head 100 sorted sql

goserver/count_tag.go

@@ -91,14 +91,9 @@ func countTagInfo(cxt *gin.Context, ret func(cxt *gin.Context, cw *tagcounter))
 	})
 
 	var other = &taginfo{Name: "Other...", Value: 0}
-	var i = 0
+
 	for _, v := range cw.CountWord {
-		if i <= 99 {
-			heap.Put(v)
-		} else {
-			break
-		}
-		i++
+		heap.Put(v)
 	}
 
 	if heap.Size() == 0 {
@@ -106,15 +101,18 @@ func countTagInfo(cxt *gin.Context, ret func(cxt *gin.Context, cw *tagcounter))
 		return
 	}
 
-	// heap.Put(other)
-	cw.PQueue = heap.Values()
+	for i := 0; i <= 100; i++ {
+		if v, ok := heap.Pop(); ok {
+			cw.PQueue = append(cw.PQueue, v)
+		}
+	}
+
 	other.Value = cw.PQueue[len(cw.PQueue)-1].(*taginfo).Value - 1
 	if other.Value == 0 {
 		other.Value = 1
 	}
 
 	cw.PQueue = append(cw.PQueue, other)
-	// cxt.JSON(200, cw.PQueue)
 	ret(cxt, cw)
 	cw.LastTime = time.Now()
 	tagCounter.Store(platform, cw)

goserver/main.go

@@ -4,6 +4,7 @@ import (
 	"database/sql"
 	"encoding/json"
 	"fmt"
+	"log"
 	"net/http"
 	"strconv"
 	"strings"
@@ -93,20 +94,37 @@ func Query(cxt *gin.Context, platform string) {
 		cxt.Error(err)
 		return
 	}
-	if psize > 100 {
-		cxt.Error(fmt.Errorf("page size <= 100"))
+	if psize > 5000 {
+		cxt.Error(fmt.Errorf("page size <= 5000"))
 		return
 	}
 
+	// filter := cxt.Query("filter")
+	orderfield := cxt.Query("orderfield")
+	ordertype := cxt.Query("ordertype")
+
+	log.Println(orderfield, ordertype)
+
 	start := (page - 1) * psize
 	// end := start + 200
 
-	ssql := fmt.Sprintf(SqlQuery, platform, strconv.Itoa(start), strconv.Itoa(psize))
+	var orderstr string
+	switch ordertype {
+	case "ascend":
+		orderstr = fmt.Sprintf("ORDER BY %s ASC", orderfield)
+	case "descend":
+		orderstr = fmt.Sprintf("ORDER BY %s DESC", orderfield)
+	default:
+		orderstr = ""
+	}
+
+	ssql := fmt.Sprintf(SqlQuery, platform, orderstr, strconv.Itoa(start), strconv.Itoa(psize))
 	rows, err := StoreStreamer.Query(ssql)
 	if err != nil {
 		cxt.Error(err)
 		return
 	}
+
 	var ots []*ObjectQuery
 	for rows.Next() {
 		ot := &ObjectQuery{}

goserver/main_test.go

@@ -42,3 +42,39 @@ func estCountTag(t *testing.T) {
 	}
 
 }
+
+func estDupTag(t *testing.T) {
+	querysql := "select uid, tags from streamer where tags is not null"
+	rows, err := StoreStreamer.Query(querysql)
+	if err != nil {
+		panic(err)
+	}
+	for rows.Next() {
+		var uid int64
+		var tagsbuf []byte
+		var tags []string
+		rows.Scan(&uid, &tagsbuf)
+		json.Unmarshal(tagsbuf, &tags)
+		if len(tags) > 0 {
+			var newtags []string
+			m := make(map[string]int)
+			for _, t := range tags {
+				if _, ok := m[t]; !ok {
+					newtags = append(newtags, t)
+					m[t] = 1
+				} else {
+					m[t]++
+				}
+			}
+			if len(newtags) != len(tags) {
+				t.Error(uid, tags)
+				newtagsbuf, err := json.Marshal(newtags)
+				if err == nil {
+					StoreStreamer.Exec("update streamer set tags = ? where uid = ?", newtagsbuf, uid)
+				} else {
+					panic(err)
+				}
+			}
+		}
+	}
+}

goserver/var_sql.go

@@ -27,8 +27,8 @@ FROM
 	intimate_extractor.streamer
 WHERE
 	platform = "%s"
-	AND operator = 0
-	AND latest_log_uid is not NULL limit %s,%s) ie
+	AND operator = 0 
+	AND latest_log_uid is not NULL ) ie
 JOIN intimate_extractor.collect_log cl
 WHERE
-ie.latest_log_uid = cl.log_uid; `
+ie.latest_log_uid = cl.log_uid %s limit %s,%s;`