From aa7749f924ed402bd670095cea0fe31658658027 Mon Sep 17 00:00:00 2001 From: huangsimin Date: Wed, 13 Jul 2022 13:33:00 +0800 Subject: [PATCH 01/35] =?UTF-8?q?=E5=AE=8C=E6=88=90=E7=8A=B6=E6=80=81?= =?UTF-8?q?=E6=9C=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/yuandian/dataflow/Server.java | 6 +- .../yuandian/dataflow/controller/TaskLog.java | 56 ++++++++------- .../dataflow/statemachine/RaftClosure.java | 27 ------- .../dataflow/statemachine/StateServer.java | 2 +- .../statemachine/SyncDataClosure.java | 45 ++++++++++++ .../dataflow/statemachine/rpc/SMResponse.java | 36 ++++++++++ .../dataflow/statemachine/rpc/SyncData.java | 72 +------------------ .../statemachine/rpc/SyncDataProcessor.java | 13 +++- .../dataflow/statemachine/rpc/TaskState.java | 42 +++++++++++ 9 files changed, 169 insertions(+), 130 deletions(-) delete mode 100644 src/main/java/com/yuandian/dataflow/statemachine/RaftClosure.java create mode 100644 src/main/java/com/yuandian/dataflow/statemachine/SyncDataClosure.java create mode 100644 src/main/java/com/yuandian/dataflow/statemachine/rpc/SMResponse.java create mode 100644 src/main/java/com/yuandian/dataflow/statemachine/rpc/TaskState.java diff --git a/src/main/java/com/yuandian/dataflow/Server.java b/src/main/java/com/yuandian/dataflow/Server.java index 9a0d35a..2962f3c 100644 --- a/src/main/java/com/yuandian/dataflow/Server.java +++ b/src/main/java/com/yuandian/dataflow/Server.java @@ -14,7 +14,7 @@ import com.alipay.sofa.jraft.RaftGroupService; import com.alipay.sofa.jraft.conf.Configuration; import com.alipay.sofa.jraft.entity.PeerId; import com.alipay.sofa.jraft.option.NodeOptions; -import com.yuandian.dataflow.statemachine.RaftClosure; +import com.yuandian.dataflow.statemachine.SyncDataClosure; import com.yuandian.dataflow.statemachine.StateMachine; import com.yuandian.dataflow.statemachine.StateServer; import com.yuandian.dataflow.statemachine.rpc.SyncDataProcessor; @@ -34,14 +34,14 @@ public class Server { @Autowired public static Node node; - public static RaftClosure done; + public static SyncDataClosure done; private static StateServer stateServer; public static Node GetNode() { return node; } - public static RaftClosure GetDone() { + public static SyncDataClosure GetDone() { return done; } diff --git a/src/main/java/com/yuandian/dataflow/controller/TaskLog.java b/src/main/java/com/yuandian/dataflow/controller/TaskLog.java index ed07bc8..e6cf27a 100644 --- a/src/main/java/com/yuandian/dataflow/controller/TaskLog.java +++ b/src/main/java/com/yuandian/dataflow/controller/TaskLog.java @@ -2,14 +2,15 @@ package com.yuandian.dataflow.controller; import java.nio.ByteBuffer; +import com.alibaba.nacos.api.naming.pojo.Cluster; +import com.alibaba.nacos.common.remote.client.RpcClientFactory; import com.alipay.sofa.jraft.Closure; import com.alipay.sofa.jraft.entity.Task; - -import com.google.protobuf.util.JsonFormat; +import com.alipay.sofa.jraft.rpc.RpcClient; +import com.alipay.sofa.jraft.rpc.impl.BoltRpcClient; import com.yuandian.dataflow.Server; -import com.yuandian.dataflow.grpc.MongodbTest; import com.yuandian.dataflow.projo.Response; -import com.yuandian.dataflow.proto.msgtype.BacktrackingFlowOuterClass; +import com.yuandian.dataflow.statemachine.SyncDataClosure; import lombok.extern.slf4j.Slf4j; @@ -25,40 +26,41 @@ import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestParam; + import com.alipay.sofa.jraft.Node; @Slf4j @Controller public class TaskLog { - // private static final Logger log = LoggerFactory.getLogger(TaskLog.class); private static Node node = Server.GetNode(); - @PostMapping(path = "/test") - public ResponseEntity Processing(@RequestBody String json) { + @GetMapping(path = "/test") + public ResponseEntity Processing() { - /*Task task = new Task(); + + log.info("node.isLeader {} {} node.getNodeId() {}", node.getNodeState(), node.getLeaderId(), node.getNodeId()); + if(node.isLeader()) { + // Task task = new Task(); - log.error(node.toString()); + // // 处理状态机 + // RaftClosure done = new RaftClosure(); + // task.setData(ByteBuffer.wrap("hello".getBytes())); + // task.setDone(done); + + // log.error("{} {} {}",node, node.toString(), task); + // node.apply(task); + + // log.error("{}", "RaftClosure"); - RaftClosure done = new RaftClosure(); - task.setData(ByteBuffer.wrap("hello".getBytes())); - task.setDone(done); - Server.GetNode().apply(task);*/ + + + } + - try { - // 1、类型转换 - BacktrackingFlowOuterClass.BacktrackingFlow.Builder builder = BacktrackingFlowOuterClass.BacktrackingFlow.newBuilder(); - JsonFormat.parser().merge(json, builder); - BacktrackingFlowOuterClass.BacktrackingFlow backtrackingFlow = builder.build(); - - // 2、业务处理 - - // 3、数据保存到 mongoDB - MongodbTest.insertMsgToMongoDB(backtrackingFlow); - } catch (Exception e) { - e.printStackTrace(); - } + + Response response = new Response(); response.Code = HttpStatus.OK; @@ -67,7 +69,7 @@ public class TaskLog { } @GetMapping(path = "/test2") - public ResponseEntity MongodbTest(@RequestBody int status) { + public ResponseEntity MongodbTest(@RequestParam int status) { Response response = new Response(); return new ResponseEntity(response, HttpStatus.OK); diff --git a/src/main/java/com/yuandian/dataflow/statemachine/RaftClosure.java b/src/main/java/com/yuandian/dataflow/statemachine/RaftClosure.java deleted file mode 100644 index 779a1a2..0000000 --- a/src/main/java/com/yuandian/dataflow/statemachine/RaftClosure.java +++ /dev/null @@ -1,27 +0,0 @@ -package com.yuandian.dataflow.statemachine; - -import com.alipay.sofa.jraft.Closure; -import com.alipay.sofa.jraft.Status; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class RaftClosure implements Closure { - - private static final Logger LOG = LoggerFactory.getLogger(StateMachine.class); - - @Override - public void run(Status status) { - - LOG.info("Task completed with status"+status.getCode()); - LOG.info("Task completed with "+status.getErrorMsg()); - LOG.info("Task completed with "+status.getRaftError()); - - } - -// @Override -// public void onCommitted() { -// System.out.println("Task onCommitted"); -// } - -} diff --git a/src/main/java/com/yuandian/dataflow/statemachine/StateServer.java b/src/main/java/com/yuandian/dataflow/statemachine/StateServer.java index 01e970c..ea617f4 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/StateServer.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/StateServer.java @@ -20,7 +20,7 @@ import com.alipay.sofa.jraft.option.NodeOptions; import com.alipay.sofa.jraft.rpc.InvokeCallback; import com.alipay.sofa.jraft.rpc.impl.BoltRaftRpcFactory; import com.alipay.sofa.jraft.util.Endpoint; -import com.yuandian.dataflow.statemachine.RaftClosure; +import com.yuandian.dataflow.statemachine.SyncDataClosure; import com.yuandian.dataflow.statemachine.StateMachine; import com.yuandian.dataflow.statemachine.rpc.SyncData; import com.yuandian.dataflow.statemachine.rpc.SyncDataProcessor; diff --git a/src/main/java/com/yuandian/dataflow/statemachine/SyncDataClosure.java b/src/main/java/com/yuandian/dataflow/statemachine/SyncDataClosure.java new file mode 100644 index 0000000..d714d2d --- /dev/null +++ b/src/main/java/com/yuandian/dataflow/statemachine/SyncDataClosure.java @@ -0,0 +1,45 @@ +package com.yuandian.dataflow.statemachine; + +import com.alipay.sofa.jraft.Closure; +import com.alipay.sofa.jraft.Status; +import com.yuandian.dataflow.statemachine.rpc.SMResponse; +import com.yuandian.dataflow.statemachine.rpc.TaskState; + +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import lombok.extern.slf4j.Slf4j; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Slf4j +@Getter +@Setter +@ToString +public abstract class SyncDataClosure implements Closure { + + // 状态机的统一响应 + private SMResponse response; + // 代表任务状态 + private TaskState state; + + + protected void failure(final String errorMsg, final String redirect) { + final SMResponse response = new SMResponse(); + response.setSuccess(false); + response.setMsg(errorMsg); + response.setRedirect(redirect); + setResponse(response); + } + + protected void success(final TaskState value) { + final SMResponse response = new SMResponse(); + response.setState(value); + response.setSuccess(true); + setResponse(response); + } + + + +} diff --git a/src/main/java/com/yuandian/dataflow/statemachine/rpc/SMResponse.java b/src/main/java/com/yuandian/dataflow/statemachine/rpc/SMResponse.java new file mode 100644 index 0000000..62c9182 --- /dev/null +++ b/src/main/java/com/yuandian/dataflow/statemachine/rpc/SMResponse.java @@ -0,0 +1,36 @@ +/** + * description + * + * @author eson + *2022年7月13日-09:07:22 + */ +package com.yuandian.dataflow.statemachine.rpc; + +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import lombok.extern.slf4j.Slf4j; + +/** + * description + * + * @author eson + *2022年7月13日-09:07:22 + */ +@Slf4j +@Getter +@Setter +@ToString +public class SMResponse { + + private TaskState state; + + + private boolean success; + /** + * redirect peer id + */ + private String redirect; + + private String msg; +} diff --git a/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncData.java b/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncData.java index 928702a..2a3a4a4 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncData.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncData.java @@ -29,76 +29,6 @@ public class SyncData implements Serializable { private static final long serialVersionUID = 1L; - private long queueSize = 0; - - - // @Getter - // @Setter - // public class IncrementAndGetRequest implements Serializable { - // private long delta; - // } - - // public class GetValueRequest implements Serializable { - // private static final long serialVersionUID = 9218253805003988802L; - - // public GetValueRequest() { - // super(); - // } - // } - - - // @Getter - // @Setter - // public class ValueResponse implements Serializable { - - // private static final long serialVersionUID = -4220017686727146773L; - - // private long value; - // private boolean success; - // /** - // * redirect peer id - // */ - // private String redirect; - - // private String errorMsg; - - // } - - // public class IncrementAndAddClosure implements Closure { - // // private CounterServer counterServer; - // private IncrementAndGetRequest request; - // private ValueResponse response; - // private Closure done; // 网络应答callback - - // public IncrementAndAddClosure(CounterServer counterServer, IncrementAndGetRequest request, ValueResponse response, - // Closure done) { - // super(); - // this.counterServer = counterServer; - // this.request = request; - // this.response = response; - // this.done = done; - // } - - // @Override - // public void run(Status status) { - // // 返回应答给客户端 - // if (this.done != null) { - // done.run(status); - // } - // } - - // public IncrementAndGetRequest getRequest() { - // return this.request; - // } - - // public void setRequest(IncrementAndGetRequest request) { - // this.request = request; - // } - - // public ValueResponse getResponse() { - // return this.response; - // } - - // } + private TaskState state; } diff --git a/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncDataProcessor.java b/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncDataProcessor.java index a682e27..6f7f149 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncDataProcessor.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncDataProcessor.java @@ -6,8 +6,10 @@ */ package com.yuandian.dataflow.statemachine.rpc; +import com.alipay.sofa.jraft.Status; import com.alipay.sofa.jraft.rpc.RpcContext; import com.alipay.sofa.jraft.rpc.RpcProcessor; +import com.yuandian.dataflow.statemachine.SyncDataClosure; import lombok.extern.slf4j.Slf4j; @@ -24,7 +26,16 @@ public class SyncDataProcessor implements RpcProcessor { public void handleRequest(RpcContext rpcCtx, SyncData request) { log.info("{}", rpcCtx); log.info("{}", request); - rpcCtx.sendResponse(null); // + + + final SyncDataClosure closure = new SyncDataClosure() { + @Override + public void run(Status status) { + rpcCtx.sendResponse(getResponse()); + } + }; + + } @Override diff --git a/src/main/java/com/yuandian/dataflow/statemachine/rpc/TaskState.java b/src/main/java/com/yuandian/dataflow/statemachine/rpc/TaskState.java new file mode 100644 index 0000000..f830599 --- /dev/null +++ b/src/main/java/com/yuandian/dataflow/statemachine/rpc/TaskState.java @@ -0,0 +1,42 @@ +/** + * description + * + * @author eson + *2022年7月13日-09:11:26 + */ +package com.yuandian.dataflow.statemachine.rpc; + +import java.io.Serializable; + +import com.alipay.sofa.jraft.entity.PeerId; + +import lombok.Getter; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; + +import java.lang.management.ManagementFactory; +import java.lang.management.OperatingSystemMXBean; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; + +/** + * 代表任务状态 暂时全局使用这个结构 + * + * @author eson + *2022年7月13日-09:11:26 + */ +@Slf4j +@Getter +@Setter +public class TaskState implements Serializable { + private static final long serialVersionUID = -1L; + + // 节点的对应peerID + private PeerId peerId; + private long taskQueueSize; + + + public static void main(String[] args) { + + } +} From 886a5ffe1aa1774ccd0adfa2ae1aabad0ceaada2 Mon Sep 17 00:00:00 2001 From: huangsimin Date: Thu, 14 Jul 2022 18:19:45 +0800 Subject: [PATCH 02/35] =?UTF-8?q?=E7=8A=B6=E6=80=81=E5=90=8C=E6=AD=A5=20TO?= =?UTF-8?q?DO:=20readIndex?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/yuandian/dataflow/Server.java | 46 ++-- .../yuandian/dataflow/controller/TaskLog.java | 35 ++-- .../proto/msgtype/UsrFlowOuterClass.java | 10 +- .../dataflow/statemachine/StateMachine.java | 42 +++- .../dataflow/statemachine/StateServer.java | 97 --------- .../statemachine/StateServerFactory.java | 198 ++++++++++++++++++ .../statemachine/SyncDataClosure.java | 7 +- .../dataflow/statemachine/rpc/SMResponse.java | 10 +- .../dataflow/statemachine/rpc/SyncData.java | 4 +- .../statemachine/rpc/SyncDataProcessor.java | 24 ++- .../dataflow/statemachine/rpc/TaskState.java | 6 +- src/main/resources/logback.xml | 2 +- .../statemachine/StateMachineTest.java | 74 +++++++ 13 files changed, 392 insertions(+), 163 deletions(-) delete mode 100644 src/main/java/com/yuandian/dataflow/statemachine/StateServer.java create mode 100644 src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java create mode 100644 src/test/java/com/yuandian/dataflow/statemachine/StateMachineTest.java diff --git a/src/main/java/com/yuandian/dataflow/Server.java b/src/main/java/com/yuandian/dataflow/Server.java index 2962f3c..6e210fa 100644 --- a/src/main/java/com/yuandian/dataflow/Server.java +++ b/src/main/java/com/yuandian/dataflow/Server.java @@ -16,10 +16,11 @@ import com.alipay.sofa.jraft.entity.PeerId; import com.alipay.sofa.jraft.option.NodeOptions; import com.yuandian.dataflow.statemachine.SyncDataClosure; import com.yuandian.dataflow.statemachine.StateMachine; -import com.yuandian.dataflow.statemachine.StateServer; +import com.yuandian.dataflow.statemachine.StateServerFactory; import com.yuandian.dataflow.statemachine.rpc.SyncDataProcessor; import lombok.var; +import lombok.extern.slf4j.Slf4j; @@ -28,26 +29,17 @@ import lombok.var; * Hello world! * */ +@Slf4j @SpringBootApplication @SpringBootConfiguration public class Server { - @Autowired - public static Node node; - public static SyncDataClosure done; - private static StateServer stateServer; - - public static Node GetNode() { - return node; - } - - public static SyncDataClosure GetDone() { - return done; - } + - public static void main(String[] args) { + public static void main(String[] args) throws Exception { + String[] peers = new String[]{"localhost:4440","localhost:4441","localhost:4442"}; String[] sprPeers = new String[]{"3440","3441","3442"}; @@ -56,11 +48,35 @@ public class Server { Configuration conf = JRaftUtils.getConfiguration("localhost:4440,localhost:4441,localhost:4442"); - stateServer = new StateServer(peeridstr, conf); + // StateServerFactory.my = new StateServerFactory(peeridstr, conf); + StateServerFactory.InitStateServer(peeridstr, conf); + // Thread printer = new Thread( new Runnable(){ + + // @Override + // public void run() { + // // TODO Auto-generated method stub + // while(true) { + // var state = StateServerFactory.getStateServer().getFsm().getTaskState(); + // log.info("{}", state); + // try { + // Thread.sleep(1000); + // } catch (InterruptedException e) { + // // TODO Auto-generated catch block + // e.printStackTrace(); + // } + // } + // } + + // } ); + + // printer.start(); + System.setProperty("server.port", sprPort); var app = SpringApplication.run(Server.class, args); app.start(); + + } } diff --git a/src/main/java/com/yuandian/dataflow/controller/TaskLog.java b/src/main/java/com/yuandian/dataflow/controller/TaskLog.java index e6cf27a..8ca4f6d 100644 --- a/src/main/java/com/yuandian/dataflow/controller/TaskLog.java +++ b/src/main/java/com/yuandian/dataflow/controller/TaskLog.java @@ -10,8 +10,11 @@ import com.alipay.sofa.jraft.rpc.RpcClient; import com.alipay.sofa.jraft.rpc.impl.BoltRpcClient; import com.yuandian.dataflow.Server; import com.yuandian.dataflow.projo.Response; +import com.yuandian.dataflow.statemachine.StateServerFactory; import com.yuandian.dataflow.statemachine.SyncDataClosure; +import com.yuandian.dataflow.statemachine.rpc.TaskState; +import lombok.var; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang.ObjectUtils.Null; @@ -29,42 +32,32 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestParam; import com.alipay.sofa.jraft.Node; +import com.alipay.sofa.jraft.Status; @Slf4j @Controller public class TaskLog { - private static Node node = Server.GetNode(); + @GetMapping(path = "/test") public ResponseEntity Processing() { - - log.info("node.isLeader {} {} node.getNodeId() {}", node.getNodeState(), node.getLeaderId(), node.getNodeId()); - if(node.isLeader()) { - // Task task = new Task(); - - // // 处理状态机 - // RaftClosure done = new RaftClosure(); - // task.setData(ByteBuffer.wrap("hello".getBytes())); - // task.setDone(done); - - // log.error("{} {} {}",node, node.toString(), task); - // node.apply(task); - - // log.error("{}", "RaftClosure"); - - - - } - + // var state = StateServerFactory.getStateServer().getFsm().getTaskState(); + var c = new SyncDataClosure() { + @Override + public void run(Status status) { + log.info(getTaskState().toString()); + } + }; + StateServerFactory.getStateServer().readIndexState(true, c ); Response response = new Response(); response.Code = HttpStatus.OK; - response.Message = HttpStatus.OK.toString(); + response.Message = "OK"; return new ResponseEntity(response, HttpStatus.OK); } diff --git a/src/main/java/com/yuandian/dataflow/proto/msgtype/UsrFlowOuterClass.java b/src/main/java/com/yuandian/dataflow/proto/msgtype/UsrFlowOuterClass.java index a829cdc..6dbf67e 100644 --- a/src/main/java/com/yuandian/dataflow/proto/msgtype/UsrFlowOuterClass.java +++ b/src/main/java/com/yuandian/dataflow/proto/msgtype/UsrFlowOuterClass.java @@ -20,7 +20,7 @@ public final class UsrFlowOuterClass { /** * 
-     *编号19
+     *编号19
      *
* * int32 table_id = 1; @@ -501,7 +501,7 @@ public final class UsrFlowOuterClass { private int tableId_; /** * 
-     *编号19
+     *编号19
      *
* * int32 table_id = 1; @@ -1793,7 +1793,7 @@ public final class UsrFlowOuterClass { private int tableId_ ; /** * 
-       *编号19
+       *编号19
        *
* * int32 table_id = 1; @@ -1805,7 +1805,7 @@ public final class UsrFlowOuterClass { } /** * 
-       *编号19
+       *编号19
        *
* * int32 table_id = 1; @@ -1820,7 +1820,7 @@ public final class UsrFlowOuterClass { } /** * 
-       *编号19
+       *编号19
        *
* * int32 table_id = 1; diff --git a/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java b/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java index c93f337..454c8ba 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java @@ -3,6 +3,7 @@ package com.yuandian.dataflow.statemachine; import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicLong; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -12,12 +13,22 @@ import com.alipay.sofa.jraft.Closure; import com.alipay.sofa.jraft.Iterator; import com.alipay.sofa.jraft.Status; import com.alipay.sofa.jraft.core.StateMachineAdapter; +import com.alipay.sofa.jraft.entity.PeerId; import com.alipay.sofa.jraft.error.RaftError; import com.alipay.sofa.jraft.error.RaftException; +import com.alipay.sofa.jraft.error.RemotingException; +import com.alipay.sofa.jraft.option.CliOptions; +import com.alipay.sofa.jraft.rpc.InvokeCallback; +import com.alipay.sofa.jraft.rpc.impl.BoltRaftRpcFactory; import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader; import com.alipay.sofa.jraft.storage.snapshot.SnapshotWriter; +import com.alipay.sofa.jraft.util.Endpoint; import com.alipay.sofa.jraft.util.Utils; +import com.yuandian.dataflow.statemachine.rpc.SMResponse; +import com.yuandian.dataflow.statemachine.rpc.SyncData; +import com.yuandian.dataflow.statemachine.rpc.TaskState; +import lombok.var; import lombok.extern.slf4j.Slf4j; /** @@ -28,6 +39,7 @@ import lombok.extern.slf4j.Slf4j; * 2018-Apr-09 4:52:31 PM */ @Slf4j + public class StateMachine extends StateMachineAdapter { // private static final Logger LOG = LoggerFactory.getLogger(StateMachine.class); @@ -35,7 +47,7 @@ public class StateMachine extends StateMachineAdapter { /** * Counter value */ - private final AtomicLong value = new AtomicLong(0); + private TaskState taskState = new TaskState(); /** * Leader term */ @@ -48,8 +60,8 @@ public class StateMachine extends StateMachineAdapter { /** * Returns current value. */ - public long getValue() { - return this.value.get(); + public TaskState getTaskState() { + return taskState; } @Override @@ -59,12 +71,22 @@ public class StateMachine extends StateMachineAdapter { if (iter.done() != null) { // This task is applied by this node, get value from closure to avoid additional // parsing. - - log.error("done:{}",iter.getData().toString()); - + var closure = (SyncDataClosure)iter.done(); + taskState = closure.getTaskState(); + log.info("SyncDataClosure(done) taskState:{} leaderTerm:{}",taskState, this.leaderTerm); + closure.success(taskState); + closure.run(Status.OK()); } else { - // Have to parse FetchAddRequest from this user log. - log.error("null:{}",iter.getData().toString()); + // Have to parse FetchAddRequest from this user log. + final ByteBuffer data = iter.getData(); + try { + taskState = SerializerManager.getSerializer(SerializerManager.Hessian2).deserialize( + data.array(), TaskState.class.getName()); + log.info("SyncDataClosure(null) taskState:{} leaderTerm:{}", taskState, this.leaderTerm); + } catch (CodecException e) { + e.printStackTrace(); + } + } iter.next(); @@ -100,4 +122,8 @@ public class StateMachine extends StateMachineAdapter { super.onLeaderStop(status); } + public static void main(String[] args) throws InterruptedException, RemotingException { + + } + } diff --git a/src/main/java/com/yuandian/dataflow/statemachine/StateServer.java b/src/main/java/com/yuandian/dataflow/statemachine/StateServer.java deleted file mode 100644 index ea617f4..0000000 --- a/src/main/java/com/yuandian/dataflow/statemachine/StateServer.java +++ /dev/null @@ -1,97 +0,0 @@ -/** - * description - * - * @author eson - *2022年7月12日-13:36:24 - */ -package com.yuandian.dataflow.statemachine; - -import java.io.File; - -import com.alibaba.nacos.common.remote.client.RpcClient; -import com.alipay.sofa.jraft.JRaftUtils; -import com.alipay.sofa.jraft.Node; -import com.alipay.sofa.jraft.RaftGroupService; -import com.alipay.sofa.jraft.conf.Configuration; -import com.alipay.sofa.jraft.entity.PeerId; -import com.alipay.sofa.jraft.error.RemotingException; -import com.alipay.sofa.jraft.option.CliOptions; -import com.alipay.sofa.jraft.option.NodeOptions; -import com.alipay.sofa.jraft.rpc.InvokeCallback; -import com.alipay.sofa.jraft.rpc.impl.BoltRaftRpcFactory; -import com.alipay.sofa.jraft.util.Endpoint; -import com.yuandian.dataflow.statemachine.SyncDataClosure; -import com.yuandian.dataflow.statemachine.StateMachine; -import com.yuandian.dataflow.statemachine.rpc.SyncData; -import com.yuandian.dataflow.statemachine.rpc.SyncDataProcessor; - -import lombok.var; -import lombok.extern.slf4j.Slf4j; - -/** - * description - * - * @author eson - *2022年7月12日-13:36:24 - */ -@Slf4j -@var -public class StateServer { - - public Node node; - public RaftGroupService cluster; - public StateMachine fsm; - - private String groupId = "dataflow"; - - public StateServer(String addr, Configuration conf) { - String[] peers = new String[]{"localhost:4440","localhost:4441","localhost:4442"}; - String[] sprPeers = new String[]{"3440","3441","3442"}; - - // var peeridstr = peers[Integer.parseInt(serverId)]; - // var sprPort = sprPeers[Integer.parseInt(args[0])]; - - // String groupId = "jraft"; - - // Configuration conf = JRaftUtils.getConfiguration("localhost:4440,localhost:4441,localhost:4442"); - - PeerId serverId = JRaftUtils.getPeerId(addr); - int port = serverId.getPort(); - - NodeOptions nodeOptions = new NodeOptions(); - nodeOptions.setElectionTimeoutMs(1000); - nodeOptions.setSnapshotLogIndexMargin(3600); - nodeOptions.setInitialConf(conf); - - File RaftDataFile = new File(String.format("./raftdata/%d", port) ); - log.info("{}",RaftDataFile.mkdirs()); - - nodeOptions.setLogUri( String.format("./raftdata/%d/logs", port) ); - nodeOptions.setRaftMetaUri(String.format("./raftdata/%d/rafts", port)); - nodeOptions.setSnapshotUri(String.format("./raftdata/%d/snaps", port)); - fsm = new StateMachine(); // 状态实例初始化 - nodeOptions.setFsm(fsm); - - cluster = new RaftGroupService(groupId, serverId, nodeOptions); - cluster.getRpcServer().registerProcessor(new SyncDataProcessor()); - - - - - node = cluster.start(); - } - - - public static void main(String[] args) throws InterruptedException, RemotingException { - var rpcClient = new BoltRaftRpcFactory().createRpcClient(); - - - rpcClient.init(new CliOptions()); - - var resp = rpcClient.invokeSync(new Endpoint("localhost",4441), new SyncData(), 5000); - log.info("{}", resp); - - // done = new RaftClosure(); - // node.shutdown(done); - } -} diff --git a/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java b/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java new file mode 100644 index 0000000..f58b2d8 --- /dev/null +++ b/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java @@ -0,0 +1,198 @@ +/** + * description + * + * @author eson + *2022年7月12日-13:36:24 + */ +package com.yuandian.dataflow.statemachine; + +import java.io.File; +import java.nio.ByteBuffer; +import java.util.concurrent.Executor; + +import com.alipay.remoting.exception.CodecException; +import com.alipay.remoting.serialization.SerializerManager; +import com.alipay.sofa.jraft.JRaftUtils; +import com.alipay.sofa.jraft.Node; +import com.alipay.sofa.jraft.RaftGroupService; +import com.alipay.sofa.jraft.Status; +import com.alipay.sofa.jraft.closure.ReadIndexClosure; +import com.alipay.sofa.jraft.conf.Configuration; +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.entity.Task; +import com.alipay.sofa.jraft.error.RaftError; +import com.alipay.sofa.jraft.error.RemotingException; +import com.alipay.sofa.jraft.option.CliOptions; +import com.alipay.sofa.jraft.option.NodeOptions; +import com.alipay.sofa.jraft.rpc.impl.BoltRaftRpcFactory; +import com.alipay.sofa.jraft.util.BytesUtil; +import com.alipay.sofa.jraft.util.Endpoint; +import com.yuandian.dataflow.statemachine.rpc.SMResponse; +import com.yuandian.dataflow.statemachine.rpc.SyncData; +import com.yuandian.dataflow.statemachine.rpc.SyncDataProcessor; +import com.yuandian.dataflow.statemachine.rpc.TaskState; + +import lombok.Getter; +import lombok.Setter; +import lombok.var; +import lombok.extern.slf4j.Slf4j; + +/** + * description + * + * @author eson + *2022年7月12日-13:36:24 + */ +@Slf4j +@var +public class StateServerFactory { + + private static StateServer ss; + // 必须初始化 + public static void InitStateServer(String peerstr, Configuration conf) throws Exception { + if(ss != null) { + throw new Exception("重复初始化 InitStateServer"); + } + ss = new StateServerFactory.StateServer(peerstr, conf); + } + + // 获取状态服务的对象 + public static StateServer getStateServer() { + return ss; + } + + @Getter + @Setter + public static class StateServer { + + private Node node; + private RaftGroupService cluster; + private StateMachine fsm; + + private String groupId = "dataflow"; + private Executor readIndexExecutor; + + public StateServer(String addr, Configuration conf) { + String[] peers = new String[]{"localhost:4440","localhost:4441","localhost:4442"}; + String[] sprPeers = new String[]{"3440","3441","3442"}; + + // var peeridstr = peers[Integer.parseInt(serverId)]; + // var sprPort = sprPeers[Integer.parseInt(args[0])]; + + // String groupId = "jraft"; + + // Configuration conf = JRaftUtils.getConfiguration("localhost:4440,localhost:4441,localhost:4442"); + + PeerId serverId = JRaftUtils.getPeerId(addr); + int port = serverId.getPort(); + + NodeOptions nodeOptions = new NodeOptions(); + nodeOptions.setElectionTimeoutMs(1000); + nodeOptions.setSnapshotLogIndexMargin(3600); + nodeOptions.setInitialConf(conf); + + File RaftDataFile = new File(String.format("./raftdata/%d", port) ); + log.info("{}",RaftDataFile.mkdirs()); + + nodeOptions.setLogUri( String.format("./raftdata/%d/logs", port) ); + nodeOptions.setRaftMetaUri(String.format("./raftdata/%d/rafts", port)); + nodeOptions.setSnapshotUri(String.format("./raftdata/%d/snaps", port)); + fsm = new StateMachine(); // 状态实例初始化 + nodeOptions.setFsm(fsm); + + cluster = new RaftGroupService(groupId, serverId, nodeOptions); + cluster.getRpcServer().registerProcessor(new SyncDataProcessor()); + + + + + node = cluster.start(); + } + + public boolean isLeader() { + return this.fsm.isLeader(); + } + + public TaskState getTaskState() { + return this.fsm.getTaskState(); + } + + public void applyState(TaskState state, SyncDataClosure closure) { + if (!ss.isLeader()) { + ss.handlerNotLeaderError(closure); + return; + } + + try { + closure.setTaskState(state); + final Task task = new Task(); + task.setData(ByteBuffer.wrap(SerializerManager.getSerializer(SerializerManager.Hessian2).serialize(state))); + task.setDone(closure); + StateServerFactory.getStateServer().getNode().apply(task); + } catch (CodecException e) { + String errorMsg = "Fail to encode TaskState"; + log.error(errorMsg, e); + closure.failure(errorMsg, PeerId.emptyPeer()); + closure.run(new Status(RaftError.EINTERNAL, errorMsg)); + } + } + + public void readIndexState(final boolean readOnlySafe, final SyncDataClosure closure) { + + if(!readOnlySafe){ + closure.success(getTaskState()); + closure.run(Status.OK()); + return; + } + + getNode().readIndex(BytesUtil.EMPTY_BYTES, new ReadIndexClosure() { + @Override + public void run(Status status, long index, byte[] reqCtx) { + if(status.isOk()){ + closure.success(getTaskState()); + closure.run(Status.OK()); + return; + } + + readIndexExecutor.execute(() -> { + if(isLeader()){ + log.debug("Fail to get value with 'ReadIndex': {}, try to applying to the state machine.", status); + applyState( getTaskState(), closure); + }else { + handlerNotLeaderError(closure); + } + }); + } + }); + } + + public SMResponse redirect() { + + final SMResponse response = new SMResponse(); + response.setSuccess(false); + if (this.node != null) { + final PeerId leader = this.node.getLeaderId(); + if (leader != null) { + response.setRedirect(leader); + } + } + return response; + + } + + public void handlerNotLeaderError(final SyncDataClosure closure) { + closure.failure("Not leader.", redirect().getRedirect()); + closure.run(new Status(RaftError.EPERM, "Not leader")); + } +} + + + public static void main(String[] args) throws InterruptedException, RemotingException { + var rpcClient = new BoltRaftRpcFactory().createRpcClient(); + + rpcClient.init(new CliOptions()); + + var resp = rpcClient.invokeSync(new Endpoint("localhost",4441), new SyncData(), 5000); + log.info("{}", resp); + } +} diff --git a/src/main/java/com/yuandian/dataflow/statemachine/SyncDataClosure.java b/src/main/java/com/yuandian/dataflow/statemachine/SyncDataClosure.java index d714d2d..f7e1ccf 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/SyncDataClosure.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/SyncDataClosure.java @@ -2,6 +2,7 @@ package com.yuandian.dataflow.statemachine; import com.alipay.sofa.jraft.Closure; import com.alipay.sofa.jraft.Status; +import com.alipay.sofa.jraft.entity.PeerId; import com.yuandian.dataflow.statemachine.rpc.SMResponse; import com.yuandian.dataflow.statemachine.rpc.TaskState; @@ -22,10 +23,10 @@ public abstract class SyncDataClosure implements Closure { // 状态机的统一响应 private SMResponse response; // 代表任务状态 - private TaskState state; + private TaskState taskState; - protected void failure(final String errorMsg, final String redirect) { + public void failure(final String errorMsg, final PeerId redirect) { final SMResponse response = new SMResponse(); response.setSuccess(false); response.setMsg(errorMsg); @@ -33,7 +34,7 @@ public abstract class SyncDataClosure implements Closure { setResponse(response); } - protected void success(final TaskState value) { + public void success(final TaskState value) { final SMResponse response = new SMResponse(); response.setState(value); response.setSuccess(true); diff --git a/src/main/java/com/yuandian/dataflow/statemachine/rpc/SMResponse.java b/src/main/java/com/yuandian/dataflow/statemachine/rpc/SMResponse.java index 62c9182..8a260f2 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/rpc/SMResponse.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/rpc/SMResponse.java @@ -6,6 +6,10 @@ */ package com.yuandian.dataflow.statemachine.rpc; +import java.io.Serializable; + +import com.alipay.sofa.jraft.entity.PeerId; + import lombok.Getter; import lombok.Setter; import lombok.ToString; @@ -21,7 +25,9 @@ import lombok.extern.slf4j.Slf4j; @Getter @Setter @ToString -public class SMResponse { +public class SMResponse implements Serializable { + + private static final long serialVersionUID = 1L; private TaskState state; @@ -30,7 +36,7 @@ public class SMResponse { /** * redirect peer id */ - private String redirect; + private PeerId redirect; private String msg; } diff --git a/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncData.java b/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncData.java index 2a3a4a4..e2349e3 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncData.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncData.java @@ -29,6 +29,8 @@ public class SyncData implements Serializable { private static final long serialVersionUID = 1L; - private TaskState state; + private TaskState taskState; + + } diff --git a/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncDataProcessor.java b/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncDataProcessor.java index 6f7f149..2181cb1 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncDataProcessor.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncDataProcessor.java @@ -6,11 +6,23 @@ */ package com.yuandian.dataflow.statemachine.rpc; +import java.nio.ByteBuffer; + +import com.alipay.remoting.exception.CodecException; +import com.alipay.remoting.serialization.SerializerManager; import com.alipay.sofa.jraft.Status; +import com.alipay.sofa.jraft.entity.Task; +import com.alipay.sofa.jraft.error.RaftError; import com.alipay.sofa.jraft.rpc.RpcContext; import com.alipay.sofa.jraft.rpc.RpcProcessor; +import com.yuandian.dataflow.statemachine.StateServerFactory; import com.yuandian.dataflow.statemachine.SyncDataClosure; +import com.yuandian.dataflow.statemachine.StateServerFactory.StateServer; +import com.alipay.sofa.jraft.entity.PeerId; +import org.apache.commons.lang.StringUtils; + +import lombok.var; import lombok.extern.slf4j.Slf4j; /** @@ -24,18 +36,18 @@ public class SyncDataProcessor implements RpcProcessor { @Override public void handleRequest(RpcContext rpcCtx, SyncData request) { - log.info("{}", rpcCtx); - log.info("{}", request); + + log.info("request: {}", request); - final SyncDataClosure closure = new SyncDataClosure() { @Override public void run(Status status) { rpcCtx.sendResponse(getResponse()); + log.info("{}", status); } }; - + StateServerFactory.getStateServer().applyState(request.getTaskState(), closure); } @Override @@ -43,8 +55,6 @@ public class SyncDataProcessor implements RpcProcessor { return SyncData.class.getName(); } - public static void main(String[] args) { - - } + } diff --git a/src/main/java/com/yuandian/dataflow/statemachine/rpc/TaskState.java b/src/main/java/com/yuandian/dataflow/statemachine/rpc/TaskState.java index f830599..af05361 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/rpc/TaskState.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/rpc/TaskState.java @@ -12,6 +12,7 @@ import com.alipay.sofa.jraft.entity.PeerId; import lombok.Getter; import lombok.Setter; +import lombok.ToString; import lombok.extern.slf4j.Slf4j; import java.lang.management.ManagementFactory; @@ -28,6 +29,7 @@ import java.lang.reflect.Modifier; @Slf4j @Getter @Setter +@ToString public class TaskState implements Serializable { private static final long serialVersionUID = -1L; @@ -36,7 +38,5 @@ public class TaskState implements Serializable { private long taskQueueSize; - public static void main(String[] args) { - - } + } diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml index 1a5cfba..269a553 100644 --- a/src/main/resources/logback.xml +++ b/src/main/resources/logback.xml @@ -3,7 +3,7 @@ class="ch.qos.logback.core.ConsoleAppender"> - %d{yyyyMMdd HH:mm:ss.SSS} %-5level%thread\(%file:%line\): %msg%n + %d{yyyyMMdd HH:mm:ss.SSS} %level %thread\(%file:%line\): %msg%n diff --git a/src/test/java/com/yuandian/dataflow/statemachine/StateMachineTest.java b/src/test/java/com/yuandian/dataflow/statemachine/StateMachineTest.java new file mode 100644 index 0000000..b8c37b9 --- /dev/null +++ b/src/test/java/com/yuandian/dataflow/statemachine/StateMachineTest.java @@ -0,0 +1,74 @@ +package com.yuandian.dataflow.statemachine; + +import java.util.concurrent.Executor; + +import org.junit.jupiter.api.Test; + +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.error.RemotingException; +import com.alipay.sofa.jraft.option.CliOptions; +import com.alipay.sofa.jraft.rpc.InvokeCallback; +import com.alipay.sofa.jraft.rpc.impl.BoltRaftRpcFactory; +import com.alipay.sofa.jraft.util.Endpoint; +import com.yuandian.dataflow.statemachine.rpc.SMResponse; +import com.yuandian.dataflow.statemachine.rpc.SyncData; +import com.yuandian.dataflow.statemachine.rpc.TaskState; + +import lombok.var; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class StateMachineTest { + @Test + void testOnApply() throws InterruptedException, RemotingException { + var rpcClient = new BoltRaftRpcFactory().createRpcClient(); + rpcClient.init(new CliOptions()); + + + var fstate = new TaskState(); + var fdata = new SyncData(); + fdata.setTaskState(fstate); + + var leader = new Endpoint("localhost",4441); + SMResponse resp = (SMResponse)rpcClient.invokeSync(leader, fdata + , 5000); + log.info("{}", resp); + if( resp != null && !resp.isSuccess() ) { + leader = resp.getRedirect().getEndpoint(); + resp = (SMResponse)rpcClient.invokeSync(resp.getRedirect().getEndpoint(), fdata + , 5000); + log.info("{}", resp); + } + + + int i = 0 ; + while(true) { + + var state = new TaskState(); + var data = new SyncData(); + data.setTaskState(state); + + state.setPeerId( PeerId.parsePeer("localhost:2222") ); + + state.setTaskQueueSize(i); + + var pi = i ; + i++; + if (i >= 1000) { + break; + } + rpcClient.invokeAsync(leader, data, new InvokeCallback() { + @Override + public void complete(Object result, Throwable err) { + // SMResponse resp = (SMResponse)result; + log.info("{} {} {}", result, err, pi); + } + + @Override + public Executor executor() { + return null; + } + } , 5000); + } + } +} From dce906a7eb7073ac6a8514b3848822e7a4215c4b Mon Sep 17 00:00:00 2001 From: huangsimin Date: Fri, 15 Jul 2022 01:00:22 +0800 Subject: [PATCH 03/35] =?UTF-8?q?=E5=AE=8C=E6=88=90=E7=8A=B6=E6=80=81?= =?UTF-8?q?=E6=9C=BA=E7=9A=84=E8=AF=BB=E5=86=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/yuandian/dataflow/Server.java | 4 -- .../yuandian/dataflow/controller/TaskLog.java | 61 ++++++++----------- .../statemachine/StateServerFactory.java | 17 ++++-- .../statemachine/SyncDataClosure.java | 8 ++- 4 files changed, 46 insertions(+), 44 deletions(-) diff --git a/src/main/java/com/yuandian/dataflow/Server.java b/src/main/java/com/yuandian/dataflow/Server.java index 6e210fa..b59fd1e 100644 --- a/src/main/java/com/yuandian/dataflow/Server.java +++ b/src/main/java/com/yuandian/dataflow/Server.java @@ -33,11 +33,7 @@ import lombok.extern.slf4j.Slf4j; @SpringBootApplication @SpringBootConfiguration public class Server { - - - - public static void main(String[] args) throws Exception { diff --git a/src/main/java/com/yuandian/dataflow/controller/TaskLog.java b/src/main/java/com/yuandian/dataflow/controller/TaskLog.java index 8ca4f6d..78c9dcb 100644 --- a/src/main/java/com/yuandian/dataflow/controller/TaskLog.java +++ b/src/main/java/com/yuandian/dataflow/controller/TaskLog.java @@ -1,14 +1,12 @@ package com.yuandian.dataflow.controller; -import java.nio.ByteBuffer; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestParam; -import com.alibaba.nacos.api.naming.pojo.Cluster; -import com.alibaba.nacos.common.remote.client.RpcClientFactory; -import com.alipay.sofa.jraft.Closure; -import com.alipay.sofa.jraft.entity.Task; -import com.alipay.sofa.jraft.rpc.RpcClient; -import com.alipay.sofa.jraft.rpc.impl.BoltRpcClient; -import com.yuandian.dataflow.Server; +import com.alipay.sofa.jraft.Status; import com.yuandian.dataflow.projo.Response; import com.yuandian.dataflow.statemachine.StateServerFactory; import com.yuandian.dataflow.statemachine.SyncDataClosure; @@ -17,23 +15,6 @@ import com.yuandian.dataflow.statemachine.rpc.TaskState; import lombok.var; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang.ObjectUtils.Null; -import org.apache.commons.logging.Log; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.http.HttpStatus; -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; -import org.springframework.stereotype.Controller; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestParam; - -import com.alipay.sofa.jraft.Node; -import com.alipay.sofa.jraft.Status; - @Slf4j @Controller public class TaskLog { @@ -41,23 +22,35 @@ public class TaskLog { @GetMapping(path = "/test") - public ResponseEntity Processing() { + public ResponseEntity Processing() throws InterruptedException { // var state = StateServerFactory.getStateServer().getFsm().getTaskState(); - var c = new SyncDataClosure() { + + + SyncDataClosure closure = new SyncDataClosure() { @Override public void run(Status status) { - log.info(getTaskState().toString()); + synchronized(lockObject) { + lockObject.notify(); + } } - }; - StateServerFactory.getStateServer().readIndexState(true, c ); - - + }; + var state = new TaskState(); + state.setTaskQueueSize(1); + closure.setTaskState(state); + StateServerFactory.getStateServer().readIndexState(true, closure); + + synchronized(closure.lockObject) { + closure.lockObject.wait(); + } + + + - Response response = new Response(); + final Response response = new Response(); response.Code = HttpStatus.OK; - response.Message = "OK"; + response.Message = closure.getTaskState().toString(); return new ResponseEntity(response, HttpStatus.OK); } diff --git a/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java b/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java index f58b2d8..25bd435 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java @@ -140,6 +140,8 @@ public class StateServerFactory { public void readIndexState(final boolean readOnlySafe, final SyncDataClosure closure) { if(!readOnlySafe){ + + closure.setTaskState(getTaskState()); closure.success(getTaskState()); closure.run(Status.OK()); return; @@ -149,15 +151,22 @@ public class StateServerFactory { @Override public void run(Status status, long index, byte[] reqCtx) { if(status.isOk()){ - closure.success(getTaskState()); - closure.run(Status.OK()); + + if(closure.getTaskState() != null){ + applyState(closure.getTaskState(), closure); + } else { + closure.setTaskState(getTaskState()); + closure.success(getTaskState()); + closure.run(Status.OK()); + } + return; } readIndexExecutor.execute(() -> { if(isLeader()){ - log.debug("Fail to get value with 'ReadIndex': {}, try to applying to the state machine.", status); - applyState( getTaskState(), closure); + log.info("Fail to get value with 'ReadIndex': {}, try to applying to the state machine.", status); + applyState(getTaskState(), closure); }else { handlerNotLeaderError(closure); } diff --git a/src/main/java/com/yuandian/dataflow/statemachine/SyncDataClosure.java b/src/main/java/com/yuandian/dataflow/statemachine/SyncDataClosure.java index f7e1ccf..7a83439 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/SyncDataClosure.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/SyncDataClosure.java @@ -25,7 +25,13 @@ public abstract class SyncDataClosure implements Closure { // 代表任务状态 private TaskState taskState; + public Object lockObject = new Object(); + public SyncDataClosure() { + + } + + public void failure(final String errorMsg, final PeerId redirect) { final SMResponse response = new SMResponse(); response.setSuccess(false); @@ -41,6 +47,4 @@ public abstract class SyncDataClosure implements Closure { setResponse(response); } - - } From 5a50ce22ef5185e965a2af95501ec56dfb2c1954 Mon Sep 17 00:00:00 2001 From: huangsimin Date: Fri, 15 Jul 2022 18:42:35 +0800 Subject: [PATCH 04/35] todo: next readIndex --- .../java/com/yuandian/dataflow/Server.java | 12 ++- .../yuandian/dataflow/controller/TaskLog.java | 32 +++---- .../dataflow/statemachine/StateMachine.java | 27 +++--- .../statemachine/StateServerFactory.java | 96 +++++++++++++------ .../statemachine/SyncDataClosure.java | 10 +- .../dataflow/statemachine/rpc/SMResponse.java | 3 +- .../statemachine/rpc/SyncDataProcessor.java | 8 +- .../{SyncData.java => SyncDataRequest.java} | 10 +- .../{rpc/TaskState.java => state/State.java} | 15 ++- .../statemachine/state/WorkerState.java | 32 +++++++ .../statemachine/StateMachineTest.java | 23 ++--- start.sh | 2 +- 12 files changed, 165 insertions(+), 105 deletions(-) rename src/main/java/com/yuandian/dataflow/statemachine/rpc/{SyncData.java => SyncDataRequest.java} (65%) rename src/main/java/com/yuandian/dataflow/statemachine/{rpc/TaskState.java => state/State.java} (68%) create mode 100644 src/main/java/com/yuandian/dataflow/statemachine/state/WorkerState.java diff --git a/src/main/java/com/yuandian/dataflow/Server.java b/src/main/java/com/yuandian/dataflow/Server.java index b59fd1e..e1fbba3 100644 --- a/src/main/java/com/yuandian/dataflow/Server.java +++ b/src/main/java/com/yuandian/dataflow/Server.java @@ -8,6 +8,7 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.SpringBootConfiguration; import org.springframework.boot.autoconfigure.SpringBootApplication; +import com.alipay.remoting.serialization.SerializerManager; import com.alipay.sofa.jraft.JRaftUtils; import com.alipay.sofa.jraft.Node; import com.alipay.sofa.jraft.RaftGroupService; @@ -37,14 +38,21 @@ public class Server { public static void main(String[] args) throws Exception { + + String[] peers = new String[]{"localhost:4440","localhost:4441","localhost:4442"}; String[] sprPeers = new String[]{"3440","3441","3442"}; + var peeridstr = peers[ Integer.parseInt(args[0] )]; var sprPort = sprPeers[Integer.parseInt(args[0] )]; + // var peeridstr = peers[2]; + // var sprPort = sprPeers[2]; + + Configuration conf = JRaftUtils.getConfiguration("localhost:4440,localhost:4441,localhost:4442"); - // StateServerFactory.my = new StateServerFactory(peeridstr, conf); + StateServerFactory.InitStateServer(peeridstr, conf); // Thread printer = new Thread( new Runnable(){ @@ -53,7 +61,7 @@ public class Server { // public void run() { // // TODO Auto-generated method stub // while(true) { - // var state = StateServerFactory.getStateServer().getFsm().getTaskState(); + // var state = StateServerFactory.getStateServer().getFsm().getState(); // log.info("{}", state); // try { // Thread.sleep(1000); diff --git a/src/main/java/com/yuandian/dataflow/controller/TaskLog.java b/src/main/java/com/yuandian/dataflow/controller/TaskLog.java index 78c9dcb..59ddb33 100644 --- a/src/main/java/com/yuandian/dataflow/controller/TaskLog.java +++ b/src/main/java/com/yuandian/dataflow/controller/TaskLog.java @@ -10,7 +10,7 @@ import com.alipay.sofa.jraft.Status; import com.yuandian.dataflow.projo.Response; import com.yuandian.dataflow.statemachine.StateServerFactory; import com.yuandian.dataflow.statemachine.SyncDataClosure; -import com.yuandian.dataflow.statemachine.rpc.TaskState; +import com.yuandian.dataflow.statemachine.state.State; import lombok.var; import lombok.extern.slf4j.Slf4j; @@ -23,34 +23,26 @@ public class TaskLog { @GetMapping(path = "/test") public ResponseEntity Processing() throws InterruptedException { - - // var state = StateServerFactory.getStateServer().getFsm().getTaskState(); - - + SyncDataClosure closure = new SyncDataClosure() { @Override public void run(Status status) { - synchronized(lockObject) { - lockObject.notify(); - } + log.info(getState().toString()); } }; - var state = new TaskState(); - state.setTaskQueueSize(1); - closure.setTaskState(state); - StateServerFactory.getStateServer().readIndexState(true, closure); - - synchronized(closure.lockObject) { - closure.lockObject.wait(); - } - - + + var state = new State(); + closure.setState(state); + + log.info(StateServerFactory.getStateServer().getFsm().getState().toString() ); + // state.getWorker().setPeerId(StateServerFactory.getStateServer().getNode().getNodeId().getPeerId()); + // state.getWorker().setTaskQueueSize(1); - + StateServerFactory.getStateServer().readIndexState(true, closure); final Response response = new Response(); response.Code = HttpStatus.OK; - response.Message = closure.getTaskState().toString(); + response.Message = "ok"; return new ResponseEntity(response, HttpStatus.OK); } diff --git a/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java b/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java index 454c8ba..9fc88e3 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java @@ -25,8 +25,8 @@ import com.alipay.sofa.jraft.storage.snapshot.SnapshotWriter; import com.alipay.sofa.jraft.util.Endpoint; import com.alipay.sofa.jraft.util.Utils; import com.yuandian.dataflow.statemachine.rpc.SMResponse; -import com.yuandian.dataflow.statemachine.rpc.SyncData; -import com.yuandian.dataflow.statemachine.rpc.TaskState; +import com.yuandian.dataflow.statemachine.rpc.SyncDataRequest; +import com.yuandian.dataflow.statemachine.state.State; import lombok.var; import lombok.extern.slf4j.Slf4j; @@ -39,15 +39,14 @@ import lombok.extern.slf4j.Slf4j; * 2018-Apr-09 4:52:31 PM */ @Slf4j - public class StateMachine extends StateMachineAdapter { // private static final Logger LOG = LoggerFactory.getLogger(StateMachine.class); /** - * Counter value + * State value 全局使用的唯一状态 */ - private TaskState taskState = new TaskState(); + private State state = new State(); /** * Leader term */ @@ -58,10 +57,10 @@ public class StateMachine extends StateMachineAdapter { } /** - * Returns current value. + * Returns current value. 只有Get 操作状态由协议流程决定 Apply */ - public TaskState getTaskState() { - return taskState; + public State getState() { + return state; } @Override @@ -72,17 +71,17 @@ public class StateMachine extends StateMachineAdapter { // This task is applied by this node, get value from closure to avoid additional // parsing. var closure = (SyncDataClosure)iter.done(); - taskState = closure.getTaskState(); - log.info("SyncDataClosure(done) taskState:{} leaderTerm:{}",taskState, this.leaderTerm); - closure.success(taskState); + state = closure.getState(); + log.info("SyncDataClosure(done) taskState:{} leaderTerm:{}",state, this.leaderTerm); + closure.success(state); closure.run(Status.OK()); } else { // Have to parse FetchAddRequest from this user log. final ByteBuffer data = iter.getData(); try { - taskState = SerializerManager.getSerializer(SerializerManager.Hessian2).deserialize( - data.array(), TaskState.class.getName()); - log.info("SyncDataClosure(null) taskState:{} leaderTerm:{}", taskState, this.leaderTerm); + state = SerializerManager.getSerializer(SerializerManager.Hessian2).deserialize( + data.array(), State.class.getName()); + log.info("SyncDataClosure(null) taskState:{} leaderTerm:{}", state, this.leaderTerm); } catch (CodecException e) { e.printStackTrace(); } diff --git a/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java b/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java index 25bd435..a811f2a 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java @@ -9,7 +9,10 @@ package com.yuandian.dataflow.statemachine; import java.io.File; import java.nio.ByteBuffer; import java.util.concurrent.Executor; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import com.alipay.remoting.NamedThreadFactory; import com.alipay.remoting.exception.CodecException; import com.alipay.remoting.serialization.SerializerManager; import com.alipay.sofa.jraft.JRaftUtils; @@ -27,10 +30,12 @@ import com.alipay.sofa.jraft.option.NodeOptions; import com.alipay.sofa.jraft.rpc.impl.BoltRaftRpcFactory; import com.alipay.sofa.jraft.util.BytesUtil; import com.alipay.sofa.jraft.util.Endpoint; +import com.alipay.sofa.jraft.util.ThreadPoolUtil; import com.yuandian.dataflow.statemachine.rpc.SMResponse; -import com.yuandian.dataflow.statemachine.rpc.SyncData; +import com.yuandian.dataflow.statemachine.rpc.SyncDataRequest; import com.yuandian.dataflow.statemachine.rpc.SyncDataProcessor; -import com.yuandian.dataflow.statemachine.rpc.TaskState; +import com.yuandian.dataflow.statemachine.state.State; +import com.yuandian.dataflow.statemachine.state.WorkerState; import lombok.Getter; import lombok.Setter; @@ -54,6 +59,27 @@ public class StateServerFactory { throw new Exception("重复初始化 InitStateServer"); } ss = new StateServerFactory.StateServer(peerstr, conf); + + + ss.readIndexState(true, new SyncDataClosure() { + @Override + public void run(Status status) { + log.info("add peerid {}", getState()); + var wstate = getState().getWorkers().get(ss.node.getNodeId().getPeerId()); + if(wstate == null) { + wstate = new WorkerState(); + getState().getWorkers().put(ss.node.getNodeId().getPeerId(), wstate); + log.info("update: {}", getState()); + ss.applyState(getState(), new SyncDataClosure() { + @Override + public void run(Status status) { + log.info("{} add workers", ss.node.getNodeId().getPeerId()); + } + } ); + } + } + + }); } // 获取状态服务的对象 @@ -70,7 +96,7 @@ public class StateServerFactory { private StateMachine fsm; private String groupId = "dataflow"; - private Executor readIndexExecutor; + private Executor readIndexExecutor = createReadIndexExecutor(); public StateServer(String addr, Configuration conf) { String[] peers = new String[]{"localhost:4440","localhost:4441","localhost:4442"}; @@ -100,31 +126,32 @@ public class StateServerFactory { fsm = new StateMachine(); // 状态实例初始化 nodeOptions.setFsm(fsm); - cluster = new RaftGroupService(groupId, serverId, nodeOptions); + cluster = new RaftGroupService(groupId, serverId, nodeOptions); cluster.getRpcServer().registerProcessor(new SyncDataProcessor()); - - - - node = cluster.start(); + + if(node.isLeader()) { + + } + } public boolean isLeader() { return this.fsm.isLeader(); } - public TaskState getTaskState() { - return this.fsm.getTaskState(); + public State getState() { + return this.fsm.getState(); } - public void applyState(TaskState state, SyncDataClosure closure) { + public void applyState(State state, SyncDataClosure closure) { if (!ss.isLeader()) { ss.handlerNotLeaderError(closure); return; } try { - closure.setTaskState(state); + closure.setState(state); final Task task = new Task(); task.setData(ByteBuffer.wrap(SerializerManager.getSerializer(SerializerManager.Hessian2).serialize(state))); task.setDone(closure); @@ -140,9 +167,8 @@ public class StateServerFactory { public void readIndexState(final boolean readOnlySafe, final SyncDataClosure closure) { if(!readOnlySafe){ - - closure.setTaskState(getTaskState()); - closure.success(getTaskState()); + closure.setState(getState()); + closure.success(getState()); closure.run(Status.OK()); return; } @@ -150,23 +176,19 @@ public class StateServerFactory { getNode().readIndex(BytesUtil.EMPTY_BYTES, new ReadIndexClosure() { @Override public void run(Status status, long index, byte[] reqCtx) { - if(status.isOk()){ - if(closure.getTaskState() != null){ - applyState(closure.getTaskState(), closure); - } else { - closure.setTaskState(getTaskState()); - closure.success(getTaskState()); - closure.run(Status.OK()); - } - + if(status.isOk()){ + log.info("readIndex {}", getState()); + closure.setState(getState()); + closure.success(getState()); + closure.run(Status.OK()); return; } readIndexExecutor.execute(() -> { if(isLeader()){ log.info("Fail to get value with 'ReadIndex': {}, try to applying to the state machine.", status); - applyState(getTaskState(), closure); + applyState(getState(), closure); }else { handlerNotLeaderError(closure); } @@ -176,7 +198,6 @@ public class StateServerFactory { } public SMResponse redirect() { - final SMResponse response = new SMResponse(); response.setSuccess(false); if (this.node != null) { @@ -186,22 +207,35 @@ public class StateServerFactory { } } return response; - } public void handlerNotLeaderError(final SyncDataClosure closure) { closure.failure("Not leader.", redirect().getRedirect()); closure.run(new Status(RaftError.EPERM, "Not leader")); } + + + private Executor createReadIndexExecutor() { + // final StoreEngineOptions opts = new StoreEngineOptions(); + // return StoreEngineHelper.createReadIndexExecutor(opts.getReadIndexCoreThreads()); + + return ThreadPoolUtil.newBuilder() // + .poolName("CounterPool") // + .enableMetric(true) // + .coreThreads(4) // + .maximumThreads(4) // + .keepAliveSeconds(60L) // + .workQueue(new SynchronousQueue<>()) // + .threadFactory(new NamedThreadFactory("CounterService", true)) // + .rejectedHandler(new ThreadPoolExecutor.AbortPolicy()) // + .build(); + } } - public static void main(String[] args) throws InterruptedException, RemotingException { var rpcClient = new BoltRaftRpcFactory().createRpcClient(); - rpcClient.init(new CliOptions()); - - var resp = rpcClient.invokeSync(new Endpoint("localhost",4441), new SyncData(), 5000); + var resp = rpcClient.invokeSync(new Endpoint("localhost",4441), new SyncDataRequest(), 5000); log.info("{}", resp); } } diff --git a/src/main/java/com/yuandian/dataflow/statemachine/SyncDataClosure.java b/src/main/java/com/yuandian/dataflow/statemachine/SyncDataClosure.java index 7a83439..3d0b891 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/SyncDataClosure.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/SyncDataClosure.java @@ -4,7 +4,7 @@ import com.alipay.sofa.jraft.Closure; import com.alipay.sofa.jraft.Status; import com.alipay.sofa.jraft.entity.PeerId; import com.yuandian.dataflow.statemachine.rpc.SMResponse; -import com.yuandian.dataflow.statemachine.rpc.TaskState; +import com.yuandian.dataflow.statemachine.state.State; import lombok.Getter; import lombok.Setter; @@ -23,15 +23,14 @@ public abstract class SyncDataClosure implements Closure { // 状态机的统一响应 private SMResponse response; // 代表任务状态 - private TaskState taskState; + private State state; - public Object lockObject = new Object(); + public Object locksync = new Object(); public SyncDataClosure() { } - public void failure(final String errorMsg, final PeerId redirect) { final SMResponse response = new SMResponse(); response.setSuccess(false); @@ -40,11 +39,10 @@ public abstract class SyncDataClosure implements Closure { setResponse(response); } - public void success(final TaskState value) { + public void success(final State value) { final SMResponse response = new SMResponse(); response.setState(value); response.setSuccess(true); setResponse(response); } - } diff --git a/src/main/java/com/yuandian/dataflow/statemachine/rpc/SMResponse.java b/src/main/java/com/yuandian/dataflow/statemachine/rpc/SMResponse.java index 8a260f2..8e28092 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/rpc/SMResponse.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/rpc/SMResponse.java @@ -9,6 +9,7 @@ package com.yuandian.dataflow.statemachine.rpc; import java.io.Serializable; import com.alipay.sofa.jraft.entity.PeerId; +import com.yuandian.dataflow.statemachine.state.State; import lombok.Getter; import lombok.Setter; @@ -29,7 +30,7 @@ public class SMResponse implements Serializable { private static final long serialVersionUID = 1L; - private TaskState state; + private State state; private boolean success; diff --git a/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncDataProcessor.java b/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncDataProcessor.java index 2181cb1..0e1f4ae 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncDataProcessor.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncDataProcessor.java @@ -32,10 +32,10 @@ import lombok.extern.slf4j.Slf4j; *2022年7月12日-11:10:54 */ @Slf4j -public class SyncDataProcessor implements RpcProcessor { +public class SyncDataProcessor implements RpcProcessor { @Override - public void handleRequest(RpcContext rpcCtx, SyncData request) { + public void handleRequest(RpcContext rpcCtx, SyncDataRequest request) { log.info("request: {}", request); @@ -47,12 +47,12 @@ public class SyncDataProcessor implements RpcProcessor { } }; - StateServerFactory.getStateServer().applyState(request.getTaskState(), closure); + StateServerFactory.getStateServer().applyState(request.getState(), closure); } @Override public String interest() { - return SyncData.class.getName(); + return SyncDataRequest.class.getName(); } diff --git a/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncData.java b/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncDataRequest.java similarity index 65% rename from src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncData.java rename to src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncDataRequest.java index e2349e3..70e7228 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncData.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncDataRequest.java @@ -9,6 +9,7 @@ package com.yuandian.dataflow.statemachine.rpc; import java.io.Serializable; import com.alipay.sofa.jraft.Closure; +import com.yuandian.dataflow.statemachine.state.State; import lombok.Getter; import lombok.Setter; @@ -16,7 +17,7 @@ import lombok.ToString; import lombok.extern.slf4j.Slf4j; /** - * description + * 同步状态时, 需要用的结构类. 新增的状态可以在 State结构里添加 * * @author eson *2022年7月11日-16:01:07 @@ -25,12 +26,9 @@ import lombok.extern.slf4j.Slf4j; @Getter @Setter @ToString -public class SyncData implements Serializable { +public class SyncDataRequest implements Serializable { private static final long serialVersionUID = 1L; + private State state; - private TaskState taskState; - - - } diff --git a/src/main/java/com/yuandian/dataflow/statemachine/rpc/TaskState.java b/src/main/java/com/yuandian/dataflow/statemachine/state/State.java similarity index 68% rename from src/main/java/com/yuandian/dataflow/statemachine/rpc/TaskState.java rename to src/main/java/com/yuandian/dataflow/statemachine/state/State.java index af05361..c48a18f 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/rpc/TaskState.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/state/State.java @@ -4,7 +4,7 @@ * @author eson *2022年7月13日-09:11:26 */ -package com.yuandian.dataflow.statemachine.rpc; +package com.yuandian.dataflow.statemachine.state; import java.io.Serializable; @@ -19,9 +19,10 @@ import java.lang.management.ManagementFactory; import java.lang.management.OperatingSystemMXBean; import java.lang.reflect.Method; import java.lang.reflect.Modifier; +import java.util.HashMap; /** - * 代表任务状态 暂时全局使用这个结构 + * 代表任务状态 暂时全局使用这个结构. 添加新增状态 * * @author eson *2022年7月13日-09:11:26 @@ -30,13 +31,9 @@ import java.lang.reflect.Modifier; @Getter @Setter @ToString -public class TaskState implements Serializable { +public class State implements Serializable { + private static final long serialVersionUID = -1L; - // 节点的对应peerID - private PeerId peerId; - private long taskQueueSize; - - - + private HashMap workers = new HashMap<>(); } diff --git a/src/main/java/com/yuandian/dataflow/statemachine/state/WorkerState.java b/src/main/java/com/yuandian/dataflow/statemachine/state/WorkerState.java new file mode 100644 index 0000000..bc7e058 --- /dev/null +++ b/src/main/java/com/yuandian/dataflow/statemachine/state/WorkerState.java @@ -0,0 +1,32 @@ +/** + * description + * + * @author eson + *2022年7月15日-10:04:00 + */ +package com.yuandian.dataflow.statemachine.state; + +import java.io.Serializable; + +import com.alipay.sofa.jraft.entity.PeerId; + +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +/** + * description + * + * @author eson + *2022年7月15日-10:04:00 + */ +@Getter +@Setter +@ToString +public class WorkerState implements Serializable { + + private static final long serialVersionUID = -1L; + // 节点的对应peerID + public PeerId peerId; + public long taskQueueSize; +} diff --git a/src/test/java/com/yuandian/dataflow/statemachine/StateMachineTest.java b/src/test/java/com/yuandian/dataflow/statemachine/StateMachineTest.java index b8c37b9..1384ebe 100644 --- a/src/test/java/com/yuandian/dataflow/statemachine/StateMachineTest.java +++ b/src/test/java/com/yuandian/dataflow/statemachine/StateMachineTest.java @@ -11,8 +11,8 @@ import com.alipay.sofa.jraft.rpc.InvokeCallback; import com.alipay.sofa.jraft.rpc.impl.BoltRaftRpcFactory; import com.alipay.sofa.jraft.util.Endpoint; import com.yuandian.dataflow.statemachine.rpc.SMResponse; -import com.yuandian.dataflow.statemachine.rpc.SyncData; -import com.yuandian.dataflow.statemachine.rpc.TaskState; +import com.yuandian.dataflow.statemachine.rpc.SyncDataRequest; +import com.yuandian.dataflow.statemachine.state.State; import lombok.var; import lombok.extern.slf4j.Slf4j; @@ -25,9 +25,9 @@ public class StateMachineTest { rpcClient.init(new CliOptions()); - var fstate = new TaskState(); - var fdata = new SyncData(); - fdata.setTaskState(fstate); + var fstate = new State(); + var fdata = new SyncDataRequest(); + fdata.setState(fstate); var leader = new Endpoint("localhost",4441); SMResponse resp = (SMResponse)rpcClient.invokeSync(leader, fdata @@ -44,20 +44,21 @@ public class StateMachineTest { int i = 0 ; while(true) { - var state = new TaskState(); - var data = new SyncData(); - data.setTaskState(state); + var state = new State(); + var request = new SyncDataRequest(); // 创建请求 + request.setState(state); // 添加请求的参数 - state.setPeerId( PeerId.parsePeer("localhost:2222") ); + var wstate = state.getWorkers(); - state.setTaskQueueSize(i); + // state.getWorker().setPeerId( PeerId.parsePeer("localhost:2222") ); + // state.getWorker().setTaskQueueSize(i); var pi = i ; i++; if (i >= 1000) { break; } - rpcClient.invokeAsync(leader, data, new InvokeCallback() { + rpcClient.invokeAsync(leader, request, new InvokeCallback() { @Override public void complete(Object result, Throwable err) { // SMResponse resp = (SMResponse)result; diff --git a/start.sh b/start.sh index 5997f9d..66d3c5d 100755 --- a/start.sh +++ b/start.sh @@ -6,4 +6,4 @@ sleep 2 screen -dmS raft-0 -L java -jar target/dataflow-1.0.0-SNAPSHOT.jar 0 screen -dmS raft-1 -L java -jar target/dataflow-1.0.0-SNAPSHOT.jar 1 -screen -dmS raft-2 -L java -jar target/dataflow-1.0.0-SNAPSHOT.jar 2 +screen -dmS raft-2 -L java -jar target/dataflow-1.0.0-SNAPSHOT.jar 2 \ No newline at end of file From 493f3d4907d4b45d9297a3895f4ec6b39e0dcf59 Mon Sep 17 00:00:00 2001 From: huangsimin Date: Mon, 18 Jul 2022 03:07:52 +0800 Subject: [PATCH 05/35] TODO --- pom.xml | 4 +- .../java/com/yuandian/dataflow/Server.java | 26 +--- .../yuandian/dataflow/controller/TaskLog.java | 10 +- .../java/com/yuandian/dataflow/projo/Doc.java | 2 +- .../dataflow/statemachine/StateMachine.java | 8 +- .../statemachine/StateServerFactory.java | 114 ++++++++++-------- ...{SyncDataClosure.java => SyncClosure.java} | 17 ++- .../statemachine/rpc/RequestCondition.java | 21 ++++ ...SyncDataRequest.java => RequestState.java} | 5 +- .../rpc/{SMResponse.java => ResponseSM.java} | 2 +- .../rpc/SyncConditionProcessor.java | 59 +++++++++ ...Processor.java => SyncStateProcessor.java} | 13 +- .../statemachine/StateMachineTest.java | 6 +- start.sh | 10 +- 14 files changed, 188 insertions(+), 109 deletions(-) rename src/main/java/com/yuandian/dataflow/statemachine/{SyncDataClosure.java => SyncClosure.java} (70%) create mode 100644 src/main/java/com/yuandian/dataflow/statemachine/rpc/RequestCondition.java rename src/main/java/com/yuandian/dataflow/statemachine/rpc/{SyncDataRequest.java => RequestState.java} (85%) rename src/main/java/com/yuandian/dataflow/statemachine/rpc/{SMResponse.java => ResponseSM.java} (93%) create mode 100644 src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncConditionProcessor.java rename src/main/java/com/yuandian/dataflow/statemachine/rpc/{SyncDataProcessor.java => SyncStateProcessor.java} (77%) diff --git a/pom.xml b/pom.xml index fc86a3e..384412f 100644 --- a/pom.xml +++ b/pom.xml @@ -192,7 +192,7 @@ 1.6.2 - + org.springframework.boot spring-boot-maven-plugin diff --git a/src/main/java/com/yuandian/dataflow/Server.java b/src/main/java/com/yuandian/dataflow/Server.java index e1fbba3..7702caa 100644 --- a/src/main/java/com/yuandian/dataflow/Server.java +++ b/src/main/java/com/yuandian/dataflow/Server.java @@ -15,10 +15,10 @@ import com.alipay.sofa.jraft.RaftGroupService; import com.alipay.sofa.jraft.conf.Configuration; import com.alipay.sofa.jraft.entity.PeerId; import com.alipay.sofa.jraft.option.NodeOptions; -import com.yuandian.dataflow.statemachine.SyncDataClosure; +import com.yuandian.dataflow.statemachine.SyncClosure; import com.yuandian.dataflow.statemachine.StateMachine; import com.yuandian.dataflow.statemachine.StateServerFactory; -import com.yuandian.dataflow.statemachine.rpc.SyncDataProcessor; +import com.yuandian.dataflow.statemachine.rpc.SyncStateProcessor; import lombok.var; import lombok.extern.slf4j.Slf4j; @@ -54,29 +54,7 @@ public class Server { Configuration conf = JRaftUtils.getConfiguration("localhost:4440,localhost:4441,localhost:4442"); StateServerFactory.InitStateServer(peeridstr, conf); - - // Thread printer = new Thread( new Runnable(){ - - // @Override - // public void run() { - // // TODO Auto-generated method stub - // while(true) { - // var state = StateServerFactory.getStateServer().getFsm().getState(); - // log.info("{}", state); - // try { - // Thread.sleep(1000); - // } catch (InterruptedException e) { - // // TODO Auto-generated catch block - // e.printStackTrace(); - // } - // } - // } - - // } ); - // printer.start(); - - System.setProperty("server.port", sprPort); var app = SpringApplication.run(Server.class, args); app.start(); diff --git a/src/main/java/com/yuandian/dataflow/controller/TaskLog.java b/src/main/java/com/yuandian/dataflow/controller/TaskLog.java index 59ddb33..443ab54 100644 --- a/src/main/java/com/yuandian/dataflow/controller/TaskLog.java +++ b/src/main/java/com/yuandian/dataflow/controller/TaskLog.java @@ -9,7 +9,7 @@ import org.springframework.web.bind.annotation.RequestParam; import com.alipay.sofa.jraft.Status; import com.yuandian.dataflow.projo.Response; import com.yuandian.dataflow.statemachine.StateServerFactory; -import com.yuandian.dataflow.statemachine.SyncDataClosure; +import com.yuandian.dataflow.statemachine.SyncClosure; import com.yuandian.dataflow.statemachine.state.State; import lombok.var; @@ -24,17 +24,17 @@ public class TaskLog { @GetMapping(path = "/test") public ResponseEntity Processing() throws InterruptedException { - SyncDataClosure closure = new SyncDataClosure() { + SyncClosure closure = new SyncClosure() { @Override public void run(Status status) { - log.info(getState().toString()); + log.error("{} {}",getValue().toString(), getValue().getWorkers().size()); } }; var state = new State(); - closure.setState(state); + closure.setValue(state); - log.info(StateServerFactory.getStateServer().getFsm().getState().toString() ); + log.error(StateServerFactory.getStateServer().getFsm().getState().toString() ); // state.getWorker().setPeerId(StateServerFactory.getStateServer().getNode().getNodeId().getPeerId()); // state.getWorker().setTaskQueueSize(1); diff --git a/src/main/java/com/yuandian/dataflow/projo/Doc.java b/src/main/java/com/yuandian/dataflow/projo/Doc.java index 8e56325..502695f 100644 --- a/src/main/java/com/yuandian/dataflow/projo/Doc.java +++ b/src/main/java/com/yuandian/dataflow/projo/Doc.java @@ -19,7 +19,7 @@ import lombok.Setter; -@BsonDiscriminator + @Getter @Setter public final class Doc extends Document { diff --git a/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java b/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java index 9fc88e3..1741024 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java @@ -24,8 +24,8 @@ import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader; import com.alipay.sofa.jraft.storage.snapshot.SnapshotWriter; import com.alipay.sofa.jraft.util.Endpoint; import com.alipay.sofa.jraft.util.Utils; -import com.yuandian.dataflow.statemachine.rpc.SMResponse; -import com.yuandian.dataflow.statemachine.rpc.SyncDataRequest; +import com.yuandian.dataflow.statemachine.rpc.ResponseSM; +import com.yuandian.dataflow.statemachine.rpc.RequestState; import com.yuandian.dataflow.statemachine.state.State; import lombok.var; @@ -70,8 +70,8 @@ public class StateMachine extends StateMachineAdapter { if (iter.done() != null) { // This task is applied by this node, get value from closure to avoid additional // parsing. - var closure = (SyncDataClosure)iter.done(); - state = closure.getState(); + var closure = (SyncClosure)iter.done(); + state = closure.getValue(); log.info("SyncDataClosure(done) taskState:{} leaderTerm:{}",state, this.leaderTerm); closure.success(state); closure.run(Status.OK()); diff --git a/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java b/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java index a811f2a..2945937 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java @@ -14,6 +14,7 @@ import java.util.concurrent.ThreadPoolExecutor; import com.alipay.remoting.NamedThreadFactory; import com.alipay.remoting.exception.CodecException; + import com.alipay.remoting.serialization.SerializerManager; import com.alipay.sofa.jraft.JRaftUtils; import com.alipay.sofa.jraft.Node; @@ -27,13 +28,15 @@ import com.alipay.sofa.jraft.error.RaftError; import com.alipay.sofa.jraft.error.RemotingException; import com.alipay.sofa.jraft.option.CliOptions; import com.alipay.sofa.jraft.option.NodeOptions; +import com.alipay.sofa.jraft.rpc.RpcClient; import com.alipay.sofa.jraft.rpc.impl.BoltRaftRpcFactory; import com.alipay.sofa.jraft.util.BytesUtil; import com.alipay.sofa.jraft.util.Endpoint; import com.alipay.sofa.jraft.util.ThreadPoolUtil; -import com.yuandian.dataflow.statemachine.rpc.SMResponse; -import com.yuandian.dataflow.statemachine.rpc.SyncDataRequest; -import com.yuandian.dataflow.statemachine.rpc.SyncDataProcessor; +import com.yuandian.dataflow.statemachine.rpc.ResponseSM; +import com.yuandian.dataflow.statemachine.rpc.RequestCondition; +import com.yuandian.dataflow.statemachine.rpc.RequestState; +import com.yuandian.dataflow.statemachine.rpc.SyncStateProcessor; import com.yuandian.dataflow.statemachine.state.State; import com.yuandian.dataflow.statemachine.state.WorkerState; @@ -53,33 +56,23 @@ import lombok.extern.slf4j.Slf4j; public class StateServerFactory { private static StateServer ss; - // 必须初始化 + public static void InitStateServer(String peerstr, Configuration conf) throws Exception { if(ss != null) { throw new Exception("重复初始化 InitStateServer"); } ss = new StateServerFactory.StateServer(peerstr, conf); + log.error("init peerid {}", ss.node.getNodeId().getPeerId()); + + ss.getNode().join(); + var request = new RequestCondition(); + request.setWorkerState( new WorkerState() ); + request.getWorkerState().setPeerId( ss.cluster.getServerId() ); + log.error("{}", ss.getNode().getNodeMetrics() ); + - - ss.readIndexState(true, new SyncDataClosure() { - @Override - public void run(Status status) { - log.info("add peerid {}", getState()); - var wstate = getState().getWorkers().get(ss.node.getNodeId().getPeerId()); - if(wstate == null) { - wstate = new WorkerState(); - getState().getWorkers().put(ss.node.getNodeId().getPeerId(), wstate); - log.info("update: {}", getState()); - ss.applyState(getState(), new SyncDataClosure() { - @Override - public void run(Status status) { - log.info("{} add workers", ss.node.getNodeId().getPeerId()); - } - } ); - } - } - - }); + ResponseSM resp = (ResponseSM)ss.getRpcClient().invokeSync(ss.getNode().getLeaderId().getEndpoint(), request, 5000); + log.info("{}", resp); } // 获取状态服务的对象 @@ -90,6 +83,10 @@ public class StateServerFactory { @Getter @Setter public static class StateServer { + + RpcClient rpcClient; + + private Node node; private RaftGroupService cluster; @@ -127,31 +124,29 @@ public class StateServerFactory { nodeOptions.setFsm(fsm); cluster = new RaftGroupService(groupId, serverId, nodeOptions); - cluster.getRpcServer().registerProcessor(new SyncDataProcessor()); + cluster.getRpcServer().registerProcessor(new SyncStateProcessor()); node = cluster.start(); - if(node.isLeader()) { - - } - + rpcClient = new BoltRaftRpcFactory().createRpcClient(); + rpcClient.init(new CliOptions()); } public boolean isLeader() { return this.fsm.isLeader(); } - public State getState() { + public State getFSMState() { return this.fsm.getState(); } - public void applyState(State state, SyncDataClosure closure) { + public void applyState(State state, SyncClosure closure) { if (!ss.isLeader()) { ss.handlerNotLeaderError(closure); return; } try { - closure.setState(state); + closure.setValue(state); final Task task = new Task(); task.setData(ByteBuffer.wrap(SerializerManager.getSerializer(SerializerManager.Hessian2).serialize(state))); task.setDone(closure); @@ -164,31 +159,52 @@ public class StateServerFactory { } } - public void readIndexState(final boolean readOnlySafe, final SyncDataClosure closure) { - - if(!readOnlySafe){ - closure.setState(getState()); - closure.success(getState()); - closure.run(Status.OK()); + public void applyWorkerState(WorkerState state, SyncClosure closure) { + if (!ss.isLeader()) { + ss.handlerNotLeaderError(closure); return; } + var wmap = getFSMState().getWorkers(); + var wstate = wmap.get(ss.node.getNodeId().getPeerId()); + if(wstate == null) { + wstate = new WorkerState(); + wmap.put(ss.node.getNodeId().getPeerId(), wstate); + log.error("update: {}", wmap.size()); + ss.applyState(getFSMState(), new SyncClosure() { + @Override + public void run(Status status) { + log.error("{} add workers", ss.node.getNodeId().getPeerId()); + } + } ); + } + } + + public void readIndexState(final boolean readOnlySafe, final SyncClosure closure) { + + closure.setValue(getFSMState()); + if(!readOnlySafe){ + + closure.success(getFSMState()); + closure.run(Status.OK()); + return; + } + getNode().readIndex(BytesUtil.EMPTY_BYTES, new ReadIndexClosure() { @Override public void run(Status status, long index, byte[] reqCtx) { if(status.isOk()){ - log.info("readIndex {}", getState()); - closure.setState(getState()); - closure.success(getState()); + log.error("readIndex {}", getFSMState()); + closure.success(getFSMState()); closure.run(Status.OK()); return; } readIndexExecutor.execute(() -> { if(isLeader()){ - log.info("Fail to get value with 'ReadIndex': {}, try to applying to the state machine.", status); - applyState(getState(), closure); + log.error("Fail to get value with 'ReadIndex': {}, try to applying to the state machine.", status); + applyState(getFSMState(), closure); }else { handlerNotLeaderError(closure); } @@ -197,8 +213,8 @@ public class StateServerFactory { }); } - public SMResponse redirect() { - final SMResponse response = new SMResponse(); + public ResponseSM redirect() { + final ResponseSM response = new ResponseSM(); response.setSuccess(false); if (this.node != null) { final PeerId leader = this.node.getLeaderId(); @@ -209,7 +225,7 @@ public class StateServerFactory { return response; } - public void handlerNotLeaderError(final SyncDataClosure closure) { + public void handlerNotLeaderError(final SyncClosure closure) { closure.failure("Not leader.", redirect().getRedirect()); closure.run(new Status(RaftError.EPERM, "Not leader")); } @@ -220,13 +236,13 @@ public class StateServerFactory { // return StoreEngineHelper.createReadIndexExecutor(opts.getReadIndexCoreThreads()); return ThreadPoolUtil.newBuilder() // - .poolName("CounterPool") // + .poolName("ReadIndexPool") // .enableMetric(true) // .coreThreads(4) // .maximumThreads(4) // .keepAliveSeconds(60L) // .workQueue(new SynchronousQueue<>()) // - .threadFactory(new NamedThreadFactory("CounterService", true)) // + .threadFactory(new NamedThreadFactory("ReadIndexService", true)) // .rejectedHandler(new ThreadPoolExecutor.AbortPolicy()) // .build(); } @@ -235,7 +251,7 @@ public class StateServerFactory { public static void main(String[] args) throws InterruptedException, RemotingException { var rpcClient = new BoltRaftRpcFactory().createRpcClient(); rpcClient.init(new CliOptions()); - var resp = rpcClient.invokeSync(new Endpoint("localhost",4441), new SyncDataRequest(), 5000); + var resp = rpcClient.invokeSync(new Endpoint("localhost",4441), new RequestState(), 5000); log.info("{}", resp); } } diff --git a/src/main/java/com/yuandian/dataflow/statemachine/SyncDataClosure.java b/src/main/java/com/yuandian/dataflow/statemachine/SyncClosure.java similarity index 70% rename from src/main/java/com/yuandian/dataflow/statemachine/SyncDataClosure.java rename to src/main/java/com/yuandian/dataflow/statemachine/SyncClosure.java index 3d0b891..f19f155 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/SyncDataClosure.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/SyncClosure.java @@ -3,7 +3,7 @@ package com.yuandian.dataflow.statemachine; import com.alipay.sofa.jraft.Closure; import com.alipay.sofa.jraft.Status; import com.alipay.sofa.jraft.entity.PeerId; -import com.yuandian.dataflow.statemachine.rpc.SMResponse; +import com.yuandian.dataflow.statemachine.rpc.ResponseSM; import com.yuandian.dataflow.statemachine.state.State; import lombok.Getter; @@ -18,21 +18,20 @@ import org.slf4j.LoggerFactory; @Getter @Setter @ToString -public abstract class SyncDataClosure implements Closure { +public abstract class SyncClosure implements Closure { // 状态机的统一响应 - private SMResponse response; + private ResponseSM response; // 代表任务状态 - private State state; + private T value; - public Object locksync = new Object(); - - public SyncDataClosure() { + + public SyncClosure() { } public void failure(final String errorMsg, final PeerId redirect) { - final SMResponse response = new SMResponse(); + final ResponseSM response = new ResponseSM(); response.setSuccess(false); response.setMsg(errorMsg); response.setRedirect(redirect); @@ -40,7 +39,7 @@ public abstract class SyncDataClosure implements Closure { } public void success(final State value) { - final SMResponse response = new SMResponse(); + final ResponseSM response = new ResponseSM(); response.setState(value); response.setSuccess(true); setResponse(response); diff --git a/src/main/java/com/yuandian/dataflow/statemachine/rpc/RequestCondition.java b/src/main/java/com/yuandian/dataflow/statemachine/rpc/RequestCondition.java new file mode 100644 index 0000000..2c67e0c --- /dev/null +++ b/src/main/java/com/yuandian/dataflow/statemachine/rpc/RequestCondition.java @@ -0,0 +1,21 @@ +package com.yuandian.dataflow.statemachine.rpc; + +import java.io.Serializable; + +import com.yuandian.dataflow.statemachine.state.WorkerState; + +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Getter +@Setter +@ToString +public class RequestCondition implements Serializable { + + private static final long serialVersionUID = 1L; + + private WorkerState workerState; +} diff --git a/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncDataRequest.java b/src/main/java/com/yuandian/dataflow/statemachine/rpc/RequestState.java similarity index 85% rename from src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncDataRequest.java rename to src/main/java/com/yuandian/dataflow/statemachine/rpc/RequestState.java index 70e7228..f9fb594 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncDataRequest.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/rpc/RequestState.java @@ -9,6 +9,7 @@ package com.yuandian.dataflow.statemachine.rpc; import java.io.Serializable; import com.alipay.sofa.jraft.Closure; +import com.alipay.sofa.jraft.entity.PeerId; import com.yuandian.dataflow.statemachine.state.State; import lombok.Getter; @@ -26,9 +27,9 @@ import lombok.extern.slf4j.Slf4j; @Getter @Setter @ToString -public class SyncDataRequest implements Serializable { +public class RequestState implements Serializable { private static final long serialVersionUID = 1L; + private State state; - } diff --git a/src/main/java/com/yuandian/dataflow/statemachine/rpc/SMResponse.java b/src/main/java/com/yuandian/dataflow/statemachine/rpc/ResponseSM.java similarity index 93% rename from src/main/java/com/yuandian/dataflow/statemachine/rpc/SMResponse.java rename to src/main/java/com/yuandian/dataflow/statemachine/rpc/ResponseSM.java index 8e28092..5e4694f 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/rpc/SMResponse.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/rpc/ResponseSM.java @@ -26,7 +26,7 @@ import lombok.extern.slf4j.Slf4j; @Getter @Setter @ToString -public class SMResponse implements Serializable { +public class ResponseSM implements Serializable { private static final long serialVersionUID = 1L; diff --git a/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncConditionProcessor.java b/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncConditionProcessor.java new file mode 100644 index 0000000..89ca91d --- /dev/null +++ b/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncConditionProcessor.java @@ -0,0 +1,59 @@ +/** + * description + * + * @author eson + *2022年7月12日-11:10:54 + */ +package com.yuandian.dataflow.statemachine.rpc; + +import java.nio.ByteBuffer; + +import com.alipay.remoting.exception.CodecException; +import com.alipay.remoting.serialization.SerializerManager; +import com.alipay.sofa.jraft.Status; +import com.alipay.sofa.jraft.entity.Task; +import com.alipay.sofa.jraft.error.RaftError; +import com.alipay.sofa.jraft.rpc.RpcContext; +import com.alipay.sofa.jraft.rpc.RpcProcessor; +import com.yuandian.dataflow.statemachine.StateServerFactory; +import com.yuandian.dataflow.statemachine.SyncClosure; +import com.yuandian.dataflow.statemachine.StateServerFactory.StateServer; +import com.alipay.sofa.jraft.entity.PeerId; + +import org.apache.commons.lang.StringUtils; + +import lombok.var; +import lombok.extern.slf4j.Slf4j; + +/** + * description + * + * @author eson + *2022年7月12日-11:10:54 + */ +@Slf4j +public class SyncConditionProcessor implements RpcProcessor { + + @Override + public void handleRequest(RpcContext rpcCtx, RequestCondition request) { + + log.info("request: {}", request); + final SyncClosure closure = new SyncClosure() { + @Override + public void run(Status status) { + rpcCtx.sendResponse(getResponse()); + log.info("{}", status); + } + }; + + StateServerFactory.getStateServer().applyWorkerState(request.getWorkerState(), closure); + } + + @Override + public String interest() { + return RequestState.class.getName(); + } + + + +} diff --git a/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncDataProcessor.java b/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncStateProcessor.java similarity index 77% rename from src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncDataProcessor.java rename to src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncStateProcessor.java index 0e1f4ae..ade1afd 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncDataProcessor.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncStateProcessor.java @@ -16,8 +16,9 @@ import com.alipay.sofa.jraft.error.RaftError; import com.alipay.sofa.jraft.rpc.RpcContext; import com.alipay.sofa.jraft.rpc.RpcProcessor; import com.yuandian.dataflow.statemachine.StateServerFactory; -import com.yuandian.dataflow.statemachine.SyncDataClosure; +import com.yuandian.dataflow.statemachine.SyncClosure; import com.yuandian.dataflow.statemachine.StateServerFactory.StateServer; +import com.yuandian.dataflow.statemachine.state.State; import com.alipay.sofa.jraft.entity.PeerId; import org.apache.commons.lang.StringUtils; @@ -32,14 +33,14 @@ import lombok.extern.slf4j.Slf4j; *2022年7月12日-11:10:54 */ @Slf4j -public class SyncDataProcessor implements RpcProcessor { +public class SyncStateProcessor implements RpcProcessor { @Override - public void handleRequest(RpcContext rpcCtx, SyncDataRequest request) { + public void handleRequest(RpcContext rpcCtx, RequestState request) { log.info("request: {}", request); - - final SyncDataClosure closure = new SyncDataClosure() { + + final SyncClosure closure = new SyncClosure() { @Override public void run(Status status) { rpcCtx.sendResponse(getResponse()); @@ -52,7 +53,7 @@ public class SyncDataProcessor implements RpcProcessor { @Override public String interest() { - return SyncDataRequest.class.getName(); + return RequestState.class.getName(); } diff --git a/src/test/java/com/yuandian/dataflow/statemachine/StateMachineTest.java b/src/test/java/com/yuandian/dataflow/statemachine/StateMachineTest.java index 1384ebe..d4dcc53 100644 --- a/src/test/java/com/yuandian/dataflow/statemachine/StateMachineTest.java +++ b/src/test/java/com/yuandian/dataflow/statemachine/StateMachineTest.java @@ -11,7 +11,7 @@ import com.alipay.sofa.jraft.rpc.InvokeCallback; import com.alipay.sofa.jraft.rpc.impl.BoltRaftRpcFactory; import com.alipay.sofa.jraft.util.Endpoint; import com.yuandian.dataflow.statemachine.rpc.SMResponse; -import com.yuandian.dataflow.statemachine.rpc.SyncDataRequest; +import com.yuandian.dataflow.statemachine.rpc.RequestState; import com.yuandian.dataflow.statemachine.state.State; import lombok.var; @@ -26,7 +26,7 @@ public class StateMachineTest { var fstate = new State(); - var fdata = new SyncDataRequest(); + var fdata = new RequestState(); fdata.setState(fstate); var leader = new Endpoint("localhost",4441); @@ -45,7 +45,7 @@ public class StateMachineTest { while(true) { var state = new State(); - var request = new SyncDataRequest(); // 创建请求 + var request = new RequestState(); // 创建请求 request.setState(state); // 添加请求的参数 var wstate = state.getWorkers(); diff --git a/start.sh b/start.sh index 66d3c5d..003f772 100755 --- a/start.sh +++ b/start.sh @@ -4,6 +4,10 @@ screen -S raft-2 -X quit sleep 2 -screen -dmS raft-0 -L java -jar target/dataflow-1.0.0-SNAPSHOT.jar 0 -screen -dmS raft-1 -L java -jar target/dataflow-1.0.0-SNAPSHOT.jar 1 -screen -dmS raft-2 -L java -jar target/dataflow-1.0.0-SNAPSHOT.jar 2 \ No newline at end of file +screen -dmS raft-0 -L java -jar target/dataflow-1.0.0-SNAPSHOT.jar 0 +screen -dmS raft-1 -L java -jar target/dataflow-1.0.0-SNAPSHOT.jar 1 +screen -dmS raft-2 -L java -jar target/dataflow-1.0.0-SNAPSHOT.jar 2 + +screen -S raft-0 -X logfile flush 1 +screen -S raft-1 -X logfile flush 1 +screen -S raft-2 -X logfile flush 1 \ No newline at end of file From 18d329910d61dadb7314b466a6e97c091ba78100 Mon Sep 17 00:00:00 2001 From: huangsimin Date: Mon, 18 Jul 2022 15:22:20 +0800 Subject: [PATCH 06/35] Save --- .../yuandian/dataflow/controller/TaskLog.java | 8 +- .../dataflow/statemachine/StateMachine.java | 56 ++++++- .../statemachine/StateServerFactory.java | 157 ++++++++++++------ .../rpc/SyncConditionProcessor.java | 6 +- .../statemachine/rpc/SyncStateProcessor.java | 2 +- .../statemachine/StateMachineTest.java | 8 +- start.sh | 9 +- 7 files changed, 176 insertions(+), 70 deletions(-) diff --git a/src/main/java/com/yuandian/dataflow/controller/TaskLog.java b/src/main/java/com/yuandian/dataflow/controller/TaskLog.java index 443ab54..06254c7 100644 --- a/src/main/java/com/yuandian/dataflow/controller/TaskLog.java +++ b/src/main/java/com/yuandian/dataflow/controller/TaskLog.java @@ -19,8 +19,6 @@ import lombok.extern.slf4j.Slf4j; @Controller public class TaskLog { - - @GetMapping(path = "/test") public ResponseEntity Processing() throws InterruptedException { @@ -33,8 +31,12 @@ public class TaskLog { var state = new State(); closure.setValue(state); + StateServerFactory.getStateServer().useState((fsmState)->{ + log.error(fsmState.toString() ); + return fsmState; + }); - log.error(StateServerFactory.getStateServer().getFsm().getState().toString() ); + // state.getWorker().setPeerId(StateServerFactory.getStateServer().getNode().getNodeId().getPeerId()); // state.getWorker().setTaskQueueSize(1); diff --git a/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java b/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java index 1741024..91234ed 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java @@ -5,6 +5,8 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Function; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.alipay.remoting.exception.CodecException; @@ -13,6 +15,7 @@ import com.alipay.sofa.jraft.Closure; import com.alipay.sofa.jraft.Iterator; import com.alipay.sofa.jraft.Status; import com.alipay.sofa.jraft.core.StateMachineAdapter; +import com.alipay.sofa.jraft.entity.LeaderChangeContext; import com.alipay.sofa.jraft.entity.PeerId; import com.alipay.sofa.jraft.error.RaftError; import com.alipay.sofa.jraft.error.RaftException; @@ -25,8 +28,10 @@ import com.alipay.sofa.jraft.storage.snapshot.SnapshotWriter; import com.alipay.sofa.jraft.util.Endpoint; import com.alipay.sofa.jraft.util.Utils; import com.yuandian.dataflow.statemachine.rpc.ResponseSM; +import com.yuandian.dataflow.statemachine.rpc.RequestCondition; import com.yuandian.dataflow.statemachine.rpc.RequestState; import com.yuandian.dataflow.statemachine.state.State; +import com.yuandian.dataflow.statemachine.state.WorkerState; import lombok.var; import lombok.extern.slf4j.Slf4j; @@ -59,11 +64,33 @@ public class StateMachine extends StateMachineAdapter { /** * Returns current value. 只有Get 操作状态由协议流程决定 Apply */ - public State getState() { - return state; + // public State getState() { + // return state; + // } + + + /** + * Returns current value. 读取修改都在这个函数域内进行 + */ + public void useState(Function dofunc) { + synchronized(this.state) { + dofunc.apply(this.state); + } + } + + /** + * Returns current value. 读取修改都在这个函数域内进行 + */ + public void updateState(Function dofunc) { + synchronized(this.state) { + var newstate = dofunc.apply(this.state); + this.state = newstate; + } } + @Override + // @SuppressWarnings("unchecked") public void onApply(final Iterator iter) { while (iter.hasNext()) { @@ -82,6 +109,7 @@ public class StateMachine extends StateMachineAdapter { state = SerializerManager.getSerializer(SerializerManager.Hessian2).deserialize( data.array(), State.class.getName()); log.info("SyncDataClosure(null) taskState:{} leaderTerm:{}", state, this.leaderTerm); + } catch (CodecException e) { e.printStackTrace(); } @@ -113,6 +141,30 @@ public class StateMachine extends StateMachineAdapter { this.leaderTerm.set(term); super.onLeaderStart(term); + + + } + + @Override + public void onStartFollowing(LeaderChangeContext ctx) { + // TODO Auto-generated method stub + super.onStartFollowing(ctx); + + var ss = StateServerFactory.getStateServer(); + var node = ss.getNode(); + + try { + var request = new RequestCondition(); + request.setWorkerState( new WorkerState() ); + request.getWorkerState().setPeerId( ss.getCluster().getServerId() ); + log.error("------------ leader id {}", node.getLeaderId()); + ResponseSM resp; + resp = (ResponseSM)StateServerFactory.getStateServer().getRpcClient().invokeSync(node.getLeaderId().getEndpoint(), request, 5000); + log.error("{}", resp); + } catch (InterruptedException | RemotingException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } } @Override diff --git a/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java b/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java index 2945937..0b54ff8 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java @@ -11,10 +11,10 @@ import java.nio.ByteBuffer; import java.util.concurrent.Executor; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadPoolExecutor; +import java.util.function.Function; import com.alipay.remoting.NamedThreadFactory; import com.alipay.remoting.exception.CodecException; - import com.alipay.remoting.serialization.SerializerManager; import com.alipay.sofa.jraft.JRaftUtils; import com.alipay.sofa.jraft.Node; @@ -33,9 +33,9 @@ import com.alipay.sofa.jraft.rpc.impl.BoltRaftRpcFactory; import com.alipay.sofa.jraft.util.BytesUtil; import com.alipay.sofa.jraft.util.Endpoint; import com.alipay.sofa.jraft.util.ThreadPoolUtil; -import com.yuandian.dataflow.statemachine.rpc.ResponseSM; -import com.yuandian.dataflow.statemachine.rpc.RequestCondition; import com.yuandian.dataflow.statemachine.rpc.RequestState; +import com.yuandian.dataflow.statemachine.rpc.ResponseSM; +import com.yuandian.dataflow.statemachine.rpc.SyncConditionProcessor; import com.yuandian.dataflow.statemachine.rpc.SyncStateProcessor; import com.yuandian.dataflow.statemachine.state.State; import com.yuandian.dataflow.statemachine.state.WorkerState; @@ -63,16 +63,7 @@ public class StateServerFactory { } ss = new StateServerFactory.StateServer(peerstr, conf); log.error("init peerid {}", ss.node.getNodeId().getPeerId()); - - ss.getNode().join(); - var request = new RequestCondition(); - request.setWorkerState( new WorkerState() ); - request.getWorkerState().setPeerId( ss.cluster.getServerId() ); - log.error("{}", ss.getNode().getNodeMetrics() ); - - ResponseSM resp = (ResponseSM)ss.getRpcClient().invokeSync(ss.getNode().getLeaderId().getEndpoint(), request, 5000); - log.info("{}", resp); } // 获取状态服务的对象 @@ -86,8 +77,6 @@ public class StateServerFactory { RpcClient rpcClient; - - private Node node; private RaftGroupService cluster; private StateMachine fsm; @@ -125,6 +114,7 @@ public class StateServerFactory { cluster = new RaftGroupService(groupId, serverId, nodeOptions); cluster.getRpcServer().registerProcessor(new SyncStateProcessor()); + cluster.getRpcServer().registerProcessor(new SyncConditionProcessor()); node = cluster.start(); rpcClient = new BoltRaftRpcFactory().createRpcClient(); @@ -135,8 +125,46 @@ public class StateServerFactory { return this.fsm.isLeader(); } - public State getFSMState() { - return this.fsm.getState(); + + public void useState(Function dofunc) { + + this.fsm.useState((fsmState)->{ + + + + SyncClosure closure = new SyncClosure() { + @Override + public void run(Status status) { + + } + }; + + + getNode().readIndex(BytesUtil.EMPTY_BYTES, new ReadIndexClosure() { + @Override + public void run(Status status, long index, byte[] reqCtx) { + if(status.isOk()){ + log.error("readIndex {}", fsmState); + closure.success(fsmState); + closure.run(Status.OK()); + return; + } + + readIndexExecutor.execute(() -> { + if(isLeader()){ + log.error("Fail to get value with 'ReadIndex': {}, try to applying to the state machine.", status); + applyState(fsmState, closure); + }else { + handlerNotLeaderError(closure); + } + }); + } + }); + + + + return null; + }); } public void applyState(State state, SyncClosure closure) { @@ -151,6 +179,7 @@ public class StateServerFactory { task.setData(ByteBuffer.wrap(SerializerManager.getSerializer(SerializerManager.Hessian2).serialize(state))); task.setDone(closure); StateServerFactory.getStateServer().getNode().apply(task); + } catch (CodecException e) { String errorMsg = "Fail to encode TaskState"; log.error(errorMsg, e); @@ -165,52 +194,72 @@ public class StateServerFactory { return; } - var wmap = getFSMState().getWorkers(); - var wstate = wmap.get(ss.node.getNodeId().getPeerId()); - if(wstate == null) { - wstate = new WorkerState(); - wmap.put(ss.node.getNodeId().getPeerId(), wstate); - log.error("update: {}", wmap.size()); - ss.applyState(getFSMState(), new SyncClosure() { - @Override - public void run(Status status) { - log.error("{} add workers", ss.node.getNodeId().getPeerId()); + useState((fsmState)->{ + var wmap = fsmState.getWorkers(); + var wstate = wmap.get(state.getPeerId()); + + if(wstate == null) { + wmap.put(state.getPeerId(), state); + try { + final Task task = new Task(); + log.error("{}", fsmState); + task.setData(ByteBuffer.wrap(SerializerManager.getSerializer(SerializerManager.Hessian2).serialize(fsmState))); + task.setDone(closure); + StateServerFactory.getStateServer().getNode().apply(task); + } catch (CodecException e) { + String errorMsg = "Fail to encode TaskState"; + log.error(errorMsg, e); + closure.failure(errorMsg, PeerId.emptyPeer()); + closure.run(new Status(RaftError.EINTERNAL, errorMsg)); } - } ); - } + } + + return fsmState; + }); + + + + } public void readIndexState(final boolean readOnlySafe, final SyncClosure closure) { - closure.setValue(getFSMState()); - if(!readOnlySafe){ - - closure.success(getFSMState()); - closure.run(Status.OK()); - return; - } - - getNode().readIndex(BytesUtil.EMPTY_BYTES, new ReadIndexClosure() { - @Override - public void run(Status status, long index, byte[] reqCtx) { - if(status.isOk()){ - log.error("readIndex {}", getFSMState()); - closure.success(getFSMState()); - closure.run(Status.OK()); - return; - } - - readIndexExecutor.execute(() -> { - if(isLeader()){ - log.error("Fail to get value with 'ReadIndex': {}, try to applying to the state machine.", status); - applyState(getFSMState(), closure); - }else { - handlerNotLeaderError(closure); - } - }); + useState((fsmState)->{ + closure.setValue(fsmState); + if(!readOnlySafe){ + + closure.success(fsmState); + closure.run(Status.OK()); + return fsmState; } + + getNode().readIndex(BytesUtil.EMPTY_BYTES, new ReadIndexClosure() { + @Override + public void run(Status status, long index, byte[] reqCtx) { + + if(status.isOk()){ + log.error("readIndex {}", fsmState); + closure.success(fsmState); + closure.run(Status.OK()); + return; + } + + readIndexExecutor.execute(() -> { + if(isLeader()){ + log.error("Fail to get value with 'ReadIndex': {}, try to applying to the state machine.", status); + applyState(fsmState, closure); + }else { + handlerNotLeaderError(closure); + } + }); + } + }); + + return fsmState; }); + + } public ResponseSM redirect() { diff --git a/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncConditionProcessor.java b/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncConditionProcessor.java index 89ca91d..fefff27 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncConditionProcessor.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncConditionProcessor.java @@ -15,9 +15,11 @@ import com.alipay.sofa.jraft.entity.Task; import com.alipay.sofa.jraft.error.RaftError; import com.alipay.sofa.jraft.rpc.RpcContext; import com.alipay.sofa.jraft.rpc.RpcProcessor; +import com.lmax.disruptor.WorkProcessor; import com.yuandian.dataflow.statemachine.StateServerFactory; import com.yuandian.dataflow.statemachine.SyncClosure; import com.yuandian.dataflow.statemachine.StateServerFactory.StateServer; +import com.yuandian.dataflow.statemachine.state.WorkerState; import com.alipay.sofa.jraft.entity.PeerId; import org.apache.commons.lang.StringUtils; @@ -38,7 +40,7 @@ public class SyncConditionProcessor implements RpcProcessor { public void handleRequest(RpcContext rpcCtx, RequestCondition request) { log.info("request: {}", request); - final SyncClosure closure = new SyncClosure() { + final SyncClosure closure = new SyncClosure() { @Override public void run(Status status) { rpcCtx.sendResponse(getResponse()); @@ -51,7 +53,7 @@ public class SyncConditionProcessor implements RpcProcessor { @Override public String interest() { - return RequestState.class.getName(); + return RequestCondition.class.getName(); } diff --git a/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncStateProcessor.java b/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncStateProcessor.java index ade1afd..5d97764 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncStateProcessor.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncStateProcessor.java @@ -40,7 +40,7 @@ public class SyncStateProcessor implements RpcProcessor { log.info("request: {}", request); - final SyncClosure closure = new SyncClosure() { + final SyncClosure closure = new SyncClosure() { @Override public void run(Status status) { rpcCtx.sendResponse(getResponse()); diff --git a/src/test/java/com/yuandian/dataflow/statemachine/StateMachineTest.java b/src/test/java/com/yuandian/dataflow/statemachine/StateMachineTest.java index d4dcc53..809cb28 100644 --- a/src/test/java/com/yuandian/dataflow/statemachine/StateMachineTest.java +++ b/src/test/java/com/yuandian/dataflow/statemachine/StateMachineTest.java @@ -10,7 +10,7 @@ import com.alipay.sofa.jraft.option.CliOptions; import com.alipay.sofa.jraft.rpc.InvokeCallback; import com.alipay.sofa.jraft.rpc.impl.BoltRaftRpcFactory; import com.alipay.sofa.jraft.util.Endpoint; -import com.yuandian.dataflow.statemachine.rpc.SMResponse; +import com.yuandian.dataflow.statemachine.rpc.ResponseSM; import com.yuandian.dataflow.statemachine.rpc.RequestState; import com.yuandian.dataflow.statemachine.state.State; @@ -30,12 +30,12 @@ public class StateMachineTest { fdata.setState(fstate); var leader = new Endpoint("localhost",4441); - SMResponse resp = (SMResponse)rpcClient.invokeSync(leader, fdata + ResponseSM resp = (ResponseSM)rpcClient.invokeSync(leader, fdata , 5000); log.info("{}", resp); if( resp != null && !resp.isSuccess() ) { leader = resp.getRedirect().getEndpoint(); - resp = (SMResponse)rpcClient.invokeSync(resp.getRedirect().getEndpoint(), fdata + resp = (ResponseSM)rpcClient.invokeSync(resp.getRedirect().getEndpoint(), fdata , 5000); log.info("{}", resp); } @@ -61,7 +61,7 @@ public class StateMachineTest { rpcClient.invokeAsync(leader, request, new InvokeCallback() { @Override public void complete(Object result, Throwable err) { - // SMResponse resp = (SMResponse)result; + // ResponseSM resp = (ResponseSM)result; log.info("{} {} {}", result, err, pi); } diff --git a/start.sh b/start.sh index 003f772..42bed0d 100755 --- a/start.sh +++ b/start.sh @@ -2,12 +2,13 @@ screen -S raft-0 -X quit screen -S raft-1 -X quit screen -S raft-2 -X quit -sleep 2 +sleep 1 screen -dmS raft-0 -L java -jar target/dataflow-1.0.0-SNAPSHOT.jar 0 screen -dmS raft-1 -L java -jar target/dataflow-1.0.0-SNAPSHOT.jar 1 screen -dmS raft-2 -L java -jar target/dataflow-1.0.0-SNAPSHOT.jar 2 -screen -S raft-0 -X logfile flush 1 -screen -S raft-1 -X logfile flush 1 -screen -S raft-2 -X logfile flush 1 \ No newline at end of file + +screen -S raft-0 -X logfile flush 0 +screen -S raft-1 -X logfile flush 0 +screen -S raft-2 -X logfile flush 0 \ No newline at end of file From ad78dd261389e10ec850774245f53aaafaa70026 Mon Sep 17 00:00:00 2001 From: huangsimin Date: Mon, 18 Jul 2022 17:54:37 +0800 Subject: [PATCH 07/35] =?UTF-8?q?TODO:=20=E6=9B=B4=E5=A5=BD=E8=AE=BE?= =?UTF-8?q?=E8=AE=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../yuandian/dataflow/controller/TaskLog.java | 4 +- .../dataflow/statemachine/StateMachine.java | 20 +++--- .../statemachine/StateServerFactory.java | 72 +++++++++++-------- 3 files changed, 54 insertions(+), 42 deletions(-) diff --git a/src/main/java/com/yuandian/dataflow/controller/TaskLog.java b/src/main/java/com/yuandian/dataflow/controller/TaskLog.java index 06254c7..1360774 100644 --- a/src/main/java/com/yuandian/dataflow/controller/TaskLog.java +++ b/src/main/java/com/yuandian/dataflow/controller/TaskLog.java @@ -31,9 +31,9 @@ public class TaskLog { var state = new State(); closure.setValue(state); - StateServerFactory.getStateServer().useState((fsmState)->{ + StateServerFactory.getStateServer().useFsmState((fsmState)->{ log.error(fsmState.toString() ); - return fsmState; + return null; }); diff --git a/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java b/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java index 91234ed..969eac1 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java @@ -64,9 +64,9 @@ public class StateMachine extends StateMachineAdapter { /** * Returns current value. 只有Get 操作状态由协议流程决定 Apply */ - // public State getState() { - // return state; - // } + public State getState() { + return state; + } /** @@ -106,10 +106,12 @@ public class StateMachine extends StateMachineAdapter { // Have to parse FetchAddRequest from this user log. final ByteBuffer data = iter.getData(); try { - state = SerializerManager.getSerializer(SerializerManager.Hessian2).deserialize( - data.array(), State.class.getName()); - log.info("SyncDataClosure(null) taskState:{} leaderTerm:{}", state, this.leaderTerm); - + + state = SerializerManager.getSerializer(SerializerManager.Hessian2).deserialize( + data.array(), State.class.getName()); + log.info("SyncDataClosure(null) taskState:{} leaderTerm:{}", state, this.leaderTerm); + + } catch (CodecException e) { e.printStackTrace(); } @@ -147,7 +149,7 @@ public class StateMachine extends StateMachineAdapter { @Override public void onStartFollowing(LeaderChangeContext ctx) { - // TODO Auto-generated method stub + super.onStartFollowing(ctx); var ss = StateServerFactory.getStateServer(); @@ -162,7 +164,7 @@ public class StateMachine extends StateMachineAdapter { resp = (ResponseSM)StateServerFactory.getStateServer().getRpcClient().invokeSync(node.getLeaderId().getEndpoint(), request, 5000); log.error("{}", resp); } catch (InterruptedException | RemotingException e) { - // TODO Auto-generated catch block + e.printStackTrace(); } } diff --git a/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java b/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java index 0b54ff8..9d0f091 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java @@ -126,48 +126,54 @@ public class StateServerFactory { } - public void useState(Function dofunc) { - - this.fsm.useState((fsmState)->{ - - - + public void useFsmState(Function dofunc) { + + SyncClosure closure = new SyncClosure() { @Override public void run(Status status) { - + } }; - + getNode().readIndex(BytesUtil.EMPTY_BYTES, new ReadIndexClosure() { @Override public void run(Status status, long index, byte[] reqCtx) { - if(status.isOk()){ - log.error("readIndex {}", fsmState); - closure.success(fsmState); - closure.run(Status.OK()); - return; - } - - readIndexExecutor.execute(() -> { - if(isLeader()){ - log.error("Fail to get value with 'ReadIndex': {}, try to applying to the state machine.", status); - applyState(fsmState, closure); - }else { - handlerNotLeaderError(closure); + + getFsm().useState((fsmState)->{ + if(status.isOk()){ + log.error("readIndex {}", fsmState); + closure.success(fsmState); + closure.run(Status.OK()); + dofunc.apply(fsmState); + return null; } + + readIndexExecutor.execute(() -> { + if(isLeader()){ + log.error("Fail to get value with 'ReadIndex': {}, try to applying to the state machine.", status); + applyState(fsmState, closure); + }else { + handlerNotLeaderError(closure); + } + }); + return null; }); + + } }); - return null; - }); + return ; + } + public void applyState(State state, SyncClosure closure) { + // 所有的提交都必须再leader进行 if (!ss.isLeader()) { ss.handlerNotLeaderError(closure); return; @@ -177,7 +183,7 @@ public class StateServerFactory { closure.setValue(state); final Task task = new Task(); task.setData(ByteBuffer.wrap(SerializerManager.getSerializer(SerializerManager.Hessian2).serialize(state))); - task.setDone(closure); + task.setDone(closure); // 确认所有数据 一致, 不需要加锁 StateServerFactory.getStateServer().getNode().apply(task); } catch (CodecException e) { @@ -194,7 +200,9 @@ public class StateServerFactory { return; } - useState((fsmState)->{ + + useFsmState((fsmState)->{ + var wmap = fsmState.getWorkers(); var wstate = wmap.get(state.getPeerId()); @@ -205,16 +213,18 @@ public class StateServerFactory { log.error("{}", fsmState); task.setData(ByteBuffer.wrap(SerializerManager.getSerializer(SerializerManager.Hessian2).serialize(fsmState))); task.setDone(closure); - StateServerFactory.getStateServer().getNode().apply(task); + StateServerFactory.getStateServer().getNode().apply(task); // 提交数据 } catch (CodecException e) { String errorMsg = "Fail to encode TaskState"; log.error(errorMsg, e); closure.failure(errorMsg, PeerId.emptyPeer()); closure.run(new Status(RaftError.EINTERNAL, errorMsg)); } + } else { + closure.success(fsmState); + closure.run(Status.OK()); } - - return fsmState; + return null; }); @@ -225,13 +235,13 @@ public class StateServerFactory { public void readIndexState(final boolean readOnlySafe, final SyncClosure closure) { - useState((fsmState)->{ + useFsmState((fsmState)->{ closure.setValue(fsmState); if(!readOnlySafe){ closure.success(fsmState); closure.run(Status.OK()); - return fsmState; + return null; } getNode().readIndex(BytesUtil.EMPTY_BYTES, new ReadIndexClosure() { @@ -256,7 +266,7 @@ public class StateServerFactory { } }); - return fsmState; + return null; }); From 36fe0e6428e65f9c167f7310bdcc1d32061a0c61 Mon Sep 17 00:00:00 2001 From: huangsimin Date: Mon, 18 Jul 2022 18:04:21 +0800 Subject: [PATCH 08/35] =?UTF-8?q?TODO:=20=E8=AE=BE=E8=AE=A1=E4=B8=80?= =?UTF-8?q?=E4=B8=AA=E7=AE=80=E5=8D=95=E6=98=93=E6=87=82api?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/yuandian/dataflow/controller/TaskLog.java | 12 ++---------- .../yuandian/dataflow/statemachine/StateMachine.java | 4 ++-- .../dataflow/statemachine/StateServerFactory.java | 1 - 3 files changed, 4 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/yuandian/dataflow/controller/TaskLog.java b/src/main/java/com/yuandian/dataflow/controller/TaskLog.java index 1360774..bfba04e 100644 --- a/src/main/java/com/yuandian/dataflow/controller/TaskLog.java +++ b/src/main/java/com/yuandian/dataflow/controller/TaskLog.java @@ -22,15 +22,7 @@ public class TaskLog { @GetMapping(path = "/test") public ResponseEntity Processing() throws InterruptedException { - SyncClosure closure = new SyncClosure() { - @Override - public void run(Status status) { - log.error("{} {}",getValue().toString(), getValue().getWorkers().size()); - } - }; - - var state = new State(); - closure.setValue(state); + StateServerFactory.getStateServer().useFsmState((fsmState)->{ log.error(fsmState.toString() ); return null; @@ -40,7 +32,7 @@ public class TaskLog { // state.getWorker().setPeerId(StateServerFactory.getStateServer().getNode().getNodeId().getPeerId()); // state.getWorker().setTaskQueueSize(1); - StateServerFactory.getStateServer().readIndexState(true, closure); + final Response response = new Response(); response.Code = HttpStatus.OK; diff --git a/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java b/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java index 969eac1..aae79a4 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java @@ -97,7 +97,7 @@ public class StateMachine extends StateMachineAdapter { if (iter.done() != null) { // This task is applied by this node, get value from closure to avoid additional // parsing. - var closure = (SyncClosure)iter.done(); + var closure = (SyncClosure)iter.done(); // 只支持单一个State. 全状态机只支持一种提交 state = closure.getValue(); log.info("SyncDataClosure(done) taskState:{} leaderTerm:{}",state, this.leaderTerm); closure.success(state); @@ -109,7 +109,7 @@ public class StateMachine extends StateMachineAdapter { state = SerializerManager.getSerializer(SerializerManager.Hessian2).deserialize( data.array(), State.class.getName()); - log.info("SyncDataClosure(null) taskState:{} leaderTerm:{}", state, this.leaderTerm); + log.info("SyncDataClosure(null) taskState:{} leaderTerm:{}", state, this.leaderTerm); } catch (CodecException e) { diff --git a/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java b/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java index 9d0f091..e1cac63 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java @@ -185,7 +185,6 @@ public class StateServerFactory { task.setData(ByteBuffer.wrap(SerializerManager.getSerializer(SerializerManager.Hessian2).serialize(state))); task.setDone(closure); // 确认所有数据 一致, 不需要加锁 StateServerFactory.getStateServer().getNode().apply(task); - } catch (CodecException e) { String errorMsg = "Fail to encode TaskState"; log.error(errorMsg, e); From 20e50d142d1c385d19409cbd034777057d84ec1e Mon Sep 17 00:00:00 2001 From: huangsimin Date: Tue, 19 Jul 2022 00:04:59 +0800 Subject: [PATCH 09/35] finish --- .../java/com/yuandian/dataflow/Server.java | 3 +- .../yuandian/dataflow/controller/TaskLog.java | 12 ++++++- .../java/com/yuandian/dataflow/projo/Doc.java | 22 ++++-------- .../dataflow/statemachine/StateMachine.java | 36 +++++++++++++++---- .../statemachine/StateServerFactory.java | 10 ++++-- .../rpc/SyncConditionProcessor.java | 3 +- .../com/yuandian/dataflow}/MongodbTest.java | 2 +- .../statemachine/StateMachineTest.java | 6 ++-- 8 files changed, 61 insertions(+), 33 deletions(-) rename src/{main/java/com/yuandian/dataflow/grpc => test/java/com/yuandian/dataflow}/MongodbTest.java (98%) diff --git a/src/main/java/com/yuandian/dataflow/Server.java b/src/main/java/com/yuandian/dataflow/Server.java index 7702caa..89ce643 100644 --- a/src/main/java/com/yuandian/dataflow/Server.java +++ b/src/main/java/com/yuandian/dataflow/Server.java @@ -7,6 +7,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.SpringApplication; import org.springframework.boot.SpringBootConfiguration; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration; import com.alipay.remoting.serialization.SerializerManager; import com.alipay.sofa.jraft.JRaftUtils; @@ -31,7 +32,7 @@ import lombok.extern.slf4j.Slf4j; * */ @Slf4j -@SpringBootApplication +@SpringBootApplication(exclude = {MongoAutoConfiguration.class}) @SpringBootConfiguration public class Server { diff --git a/src/main/java/com/yuandian/dataflow/controller/TaskLog.java b/src/main/java/com/yuandian/dataflow/controller/TaskLog.java index bfba04e..4fbf2c8 100644 --- a/src/main/java/com/yuandian/dataflow/controller/TaskLog.java +++ b/src/main/java/com/yuandian/dataflow/controller/TaskLog.java @@ -7,10 +7,13 @@ import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestParam; import com.alipay.sofa.jraft.Status; +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.error.RemotingException; import com.yuandian.dataflow.projo.Response; import com.yuandian.dataflow.statemachine.StateServerFactory; import com.yuandian.dataflow.statemachine.SyncClosure; import com.yuandian.dataflow.statemachine.state.State; +import com.yuandian.dataflow.statemachine.state.WorkerState; import lombok.var; import lombok.extern.slf4j.Slf4j; @@ -20,7 +23,7 @@ import lombok.extern.slf4j.Slf4j; public class TaskLog { @GetMapping(path = "/test") - public ResponseEntity Processing() throws InterruptedException { + public ResponseEntity Processing() throws InterruptedException, RemotingException { StateServerFactory.getStateServer().useFsmState((fsmState)->{ @@ -31,6 +34,13 @@ public class TaskLog { // state.getWorker().setPeerId(StateServerFactory.getStateServer().getNode().getNodeId().getPeerId()); // state.getWorker().setTaskQueueSize(1); + + StateServerFactory.getStateServer().updateFsmState((fsmState)->{ + log.error(fsmState.toString() ); + fsmState.getWorkers().put(new PeerId(), new WorkerState()); + return fsmState; + }); + diff --git a/src/main/java/com/yuandian/dataflow/projo/Doc.java b/src/main/java/com/yuandian/dataflow/projo/Doc.java index 502695f..e1389a1 100644 --- a/src/main/java/com/yuandian/dataflow/projo/Doc.java +++ b/src/main/java/com/yuandian/dataflow/projo/Doc.java @@ -2,27 +2,20 @@ package com.yuandian.dataflow.projo; -import java.io.Serializable; -import java.lang.annotation.Documented; import java.time.LocalDateTime; import org.bson.Document; -import org.bson.codecs.pojo.annotations.BsonCreator; -import org.bson.codecs.pojo.annotations.BsonDiscriminator; import org.bson.codecs.pojo.annotations.BsonProperty; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.util.JSONPObject; import lombok.Getter; import lombok.Setter; - - -@Getter @Setter -public final class Doc extends Document { +@Getter +public final class Doc extends Document { @JsonProperty("code") @BsonProperty("code") @@ -33,14 +26,11 @@ public final class Doc extends Document { public LocalDateTime TS; @JsonProperty("desc") - @BsonProperty("desc") + @BsonProperty("desc") public String Desc; @JsonProperty("data") - @BsonProperty("data") - public Document Data; - - - - + @BsonProperty("data") + public Document Data; + } diff --git a/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java b/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java index aae79a4..ec27c09 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java @@ -80,11 +80,32 @@ public class StateMachine extends StateMachineAdapter { /** * Returns current value. 读取修改都在这个函数域内进行 + * @throws RemotingException + * @throws InterruptedException */ - public void updateState(Function dofunc) { + public void updateState(Function dofunc) throws InterruptedException, RemotingException { synchronized(this.state) { var newstate = dofunc.apply(this.state); - this.state = newstate; + var ss = StateServerFactory.getStateServer(); + if(!isLeader()) { + var request = new RequestState(); + request.setState(newstate); + var result = ss.getRpcClient().invokeSync(ss.getNode().getLeaderId().getEndpoint(), request, 5000); + log.info("{}", result); + return; + } + + // this.state = newstate; + if(newstate != null) { + var colsure = new SyncClosure() { + @Override + public void run(Status status) { + + } + }; + colsure.setValue(newstate); + StateServerFactory.getStateServer().applyState(newstate, colsure); + } } } @@ -98,19 +119,22 @@ public class StateMachine extends StateMachineAdapter { // This task is applied by this node, get value from closure to avoid additional // parsing. var closure = (SyncClosure)iter.done(); // 只支持单一个State. 全状态机只支持一种提交 - state = closure.getValue(); - log.info("SyncDataClosure(done) taskState:{} leaderTerm:{}",state, this.leaderTerm); + log.info("SyncDataClosure(done) taskState:{} leaderTerm:{} {}",this.state, this.leaderTerm, closure); + synchronized(this.state) { + this.state = closure.getValue(); + } + closure.success(state); closure.run(Status.OK()); } else { // Have to parse FetchAddRequest from this user log. final ByteBuffer data = iter.getData(); try { - + synchronized(state) { state = SerializerManager.getSerializer(SerializerManager.Hessian2).deserialize( data.array(), State.class.getName()); log.info("SyncDataClosure(null) taskState:{} leaderTerm:{}", state, this.leaderTerm); - + } } catch (CodecException e) { e.printStackTrace(); diff --git a/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java b/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java index e1cac63..edcc8aa 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java @@ -171,6 +171,10 @@ public class StateServerFactory { } + public void updateFsmState(Function dofunc) throws InterruptedException, RemotingException { + this.getFsm().updateState(dofunc); + } + public void applyState(State state, SyncClosure closure) { // 所有的提交都必须再leader进行 @@ -193,7 +197,7 @@ public class StateServerFactory { } } - public void applyWorkerState(WorkerState state, SyncClosure closure) { + public void applyWorkerState(WorkerState state, SyncClosure closure) { if (!ss.isLeader()) { ss.handlerNotLeaderError(closure); return; @@ -209,13 +213,13 @@ public class StateServerFactory { wmap.put(state.getPeerId(), state); try { final Task task = new Task(); - log.error("{}", fsmState); + closure.setValue(fsmState); task.setData(ByteBuffer.wrap(SerializerManager.getSerializer(SerializerManager.Hessian2).serialize(fsmState))); task.setDone(closure); StateServerFactory.getStateServer().getNode().apply(task); // 提交数据 } catch (CodecException e) { String errorMsg = "Fail to encode TaskState"; - log.error(errorMsg, e); + log.error("{}:{}",errorMsg, e); closure.failure(errorMsg, PeerId.emptyPeer()); closure.run(new Status(RaftError.EINTERNAL, errorMsg)); } diff --git a/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncConditionProcessor.java b/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncConditionProcessor.java index fefff27..263a63d 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncConditionProcessor.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncConditionProcessor.java @@ -19,6 +19,7 @@ import com.lmax.disruptor.WorkProcessor; import com.yuandian.dataflow.statemachine.StateServerFactory; import com.yuandian.dataflow.statemachine.SyncClosure; import com.yuandian.dataflow.statemachine.StateServerFactory.StateServer; +import com.yuandian.dataflow.statemachine.state.State; import com.yuandian.dataflow.statemachine.state.WorkerState; import com.alipay.sofa.jraft.entity.PeerId; @@ -40,7 +41,7 @@ public class SyncConditionProcessor implements RpcProcessor { public void handleRequest(RpcContext rpcCtx, RequestCondition request) { log.info("request: {}", request); - final SyncClosure closure = new SyncClosure() { + final SyncClosure closure = new SyncClosure< State>() { @Override public void run(Status status) { rpcCtx.sendResponse(getResponse()); diff --git a/src/main/java/com/yuandian/dataflow/grpc/MongodbTest.java b/src/test/java/com/yuandian/dataflow/MongodbTest.java similarity index 98% rename from src/main/java/com/yuandian/dataflow/grpc/MongodbTest.java rename to src/test/java/com/yuandian/dataflow/MongodbTest.java index d250df0..73e1ed9 100644 --- a/src/main/java/com/yuandian/dataflow/grpc/MongodbTest.java +++ b/src/test/java/com/yuandian/dataflow/MongodbTest.java @@ -1,4 +1,4 @@ -package com.yuandian.dataflow.grpc; +package com.yuandian.dataflow; import com.mongodb.MongoClient; import com.mongodb.MongoCredential; diff --git a/src/test/java/com/yuandian/dataflow/statemachine/StateMachineTest.java b/src/test/java/com/yuandian/dataflow/statemachine/StateMachineTest.java index 809cb28..329abfa 100644 --- a/src/test/java/com/yuandian/dataflow/statemachine/StateMachineTest.java +++ b/src/test/java/com/yuandian/dataflow/statemachine/StateMachineTest.java @@ -23,8 +23,7 @@ public class StateMachineTest { void testOnApply() throws InterruptedException, RemotingException { var rpcClient = new BoltRaftRpcFactory().createRpcClient(); rpcClient.init(new CliOptions()); - - + var fstate = new State(); var fdata = new RequestState(); fdata.setState(fstate); @@ -39,8 +38,7 @@ public class StateMachineTest { , 5000); log.info("{}", resp); } - - + int i = 0 ; while(true) { From 6f743686a141423036e1304250930837cbd25ee4 Mon Sep 17 00:00:00 2001 From: huangsimin Date: Tue, 19 Jul 2022 10:15:54 +0800 Subject: [PATCH 10/35] =?UTF-8?q?=E9=80=BB=E8=BE=91=E8=B7=91=E9=80=9A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../yuandian/dataflow/controller/TaskLog.java | 7 +--- .../com/yuandian/dataflow/master/Header.java | 10 ++--- .../dataflow/statemachine/StateMachine.java | 37 ++++++++++++------- .../statemachine/StateServerFactory.java | 18 +++++---- src/main/resources/logback.xml | 15 +++++++- 5 files changed, 55 insertions(+), 32 deletions(-) diff --git a/src/main/java/com/yuandian/dataflow/controller/TaskLog.java b/src/main/java/com/yuandian/dataflow/controller/TaskLog.java index 4fbf2c8..01ae05a 100644 --- a/src/main/java/com/yuandian/dataflow/controller/TaskLog.java +++ b/src/main/java/com/yuandian/dataflow/controller/TaskLog.java @@ -27,7 +27,7 @@ public class TaskLog { StateServerFactory.getStateServer().useFsmState((fsmState)->{ - log.error(fsmState.toString() ); + log.debug(fsmState.toString() ); return null; }); @@ -36,14 +36,11 @@ public class TaskLog { // state.getWorker().setTaskQueueSize(1); StateServerFactory.getStateServer().updateFsmState((fsmState)->{ - log.error(fsmState.toString() ); + log.debug(fsmState.toString() ); fsmState.getWorkers().put(new PeerId(), new WorkerState()); return fsmState; }); - - - final Response response = new Response(); response.Code = HttpStatus.OK; response.Message = "ok"; diff --git a/src/main/java/com/yuandian/dataflow/master/Header.java b/src/main/java/com/yuandian/dataflow/master/Header.java index f1df546..d31fdc6 100644 --- a/src/main/java/com/yuandian/dataflow/master/Header.java +++ b/src/main/java/com/yuandian/dataflow/master/Header.java @@ -44,12 +44,12 @@ public class Header { var out = new DataOutputStream(sock.getOutputStream()); // 发送验证字符串 // out.write("public".getBytes()); - // log.error("{}", PacketHeader.PacketCode(in)); + // log.debug("{}", PacketHeader.PacketCode(in)); // var pheader = new PacketHeader(in); - // log.error("{}", pheader); + // log.debug("{}", pheader); // var pbase = PacketBase.createPacketBase(pheader); - // log.error("{}",pbase); + // log.debug("{}",pbase); // //60010流需要解压 @@ -74,7 +74,7 @@ public class Header { // var length = ByteBuffer.wrap(in.readNBytes(4)).order(ByteOrder.LITTLE_ENDIAN).getInt(); // if (length <= 13) { - // log.error("[{}:{}][数据总条数:{}][当前解析第{}条][length<13][探针发包有问题]"); + // log.debug("[{}:{}][数据总条数:{}][当前解析第{}条][length<13][探针发包有问题]"); // throw new Exception("数据解析异常"); // } @@ -90,7 +90,7 @@ public class Header { // if (length <= 13) { - // log.error("[{}:{}][数据总条数:{}][当前解析第{}条][length<13][探针发包有问题]"); + // log.debug("[{}:{}][数据总条数:{}][当前解析第{}条][length<13][探针发包有问题]"); // throw new Exception("数据解析异常"); // } diff --git a/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java b/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java index ec27c09..8019d81 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java @@ -153,7 +153,7 @@ public class StateMachine extends StateMachineAdapter { @Override public void onError(final RaftException e) { - log.error("Raft error: {}", e, e); + log.debug("Raft error: {}", e, e); } @Override @@ -167,30 +167,41 @@ public class StateMachine extends StateMachineAdapter { this.leaderTerm.set(term); super.onLeaderStart(term); - - + var ss = StateServerFactory.getStateServer(); + var ws = new WorkerState(); + ws.setPeerId(ss.getCluster().getServerId()); + final SyncClosure closure = new SyncClosure< State>() { + @Override + public void run(Status status) { + log.debug("leader set WorkerState {} ", status); + } + }; + ss.applyWorkerState(ws, closure); + return; } @Override public void onStartFollowing(LeaderChangeContext ctx) { super.onStartFollowing(ctx); - - var ss = StateServerFactory.getStateServer(); - var node = ss.getNode(); - try { + var ss = StateServerFactory.getStateServer(); + var ws = new WorkerState(); + ws.setPeerId(ss.getCluster().getServerId()); var request = new RequestCondition(); - request.setWorkerState( new WorkerState() ); - request.getWorkerState().setPeerId( ss.getCluster().getServerId() ); - log.error("------------ leader id {}", node.getLeaderId()); + request.setWorkerState(ws); + log.debug("my: {} leader id {}",ss.getCluster().getServerId(), ss.getNode().getLeaderId()); ResponseSM resp; - resp = (ResponseSM)StateServerFactory.getStateServer().getRpcClient().invokeSync(node.getLeaderId().getEndpoint(), request, 5000); - log.error("{}", resp); + resp = (ResponseSM)StateServerFactory.getStateServer().getRpcClient().invokeSync(ss.getNode().getLeaderId().getEndpoint(), request, 5000); + if(resp == null) { + log.error("{} set WorkerState is error", resp); + } + log.debug("WorkerState is {}", resp); + return; } catch (InterruptedException | RemotingException e) { - e.printStackTrace(); } + } @Override diff --git a/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java b/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java index edcc8aa..93173ea 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java @@ -62,7 +62,7 @@ public class StateServerFactory { throw new Exception("重复初始化 InitStateServer"); } ss = new StateServerFactory.StateServer(peerstr, conf); - log.error("init peerid {}", ss.node.getNodeId().getPeerId()); + log.debug("init peerid {}", ss.node.getNodeId().getPeerId()); } @@ -143,7 +143,7 @@ public class StateServerFactory { getFsm().useState((fsmState)->{ if(status.isOk()){ - log.error("readIndex {}", fsmState); + log.debug("readIndex {}", fsmState); closure.success(fsmState); closure.run(Status.OK()); dofunc.apply(fsmState); @@ -152,7 +152,7 @@ public class StateServerFactory { readIndexExecutor.execute(() -> { if(isLeader()){ - log.error("Fail to get value with 'ReadIndex': {}, try to applying to the state machine.", status); + log.debug("Fail to get value with 'ReadIndex': {}, try to applying to the state machine.", getNode()); applyState(fsmState, closure); }else { handlerNotLeaderError(closure); @@ -176,6 +176,8 @@ public class StateServerFactory { } + + public void applyState(State state, SyncClosure closure) { // 所有的提交都必须再leader进行 if (!ss.isLeader()) { @@ -191,7 +193,7 @@ public class StateServerFactory { StateServerFactory.getStateServer().getNode().apply(task); } catch (CodecException e) { String errorMsg = "Fail to encode TaskState"; - log.error(errorMsg, e); + log.debug(errorMsg, e); closure.failure(errorMsg, PeerId.emptyPeer()); closure.run(new Status(RaftError.EINTERNAL, errorMsg)); } @@ -219,7 +221,7 @@ public class StateServerFactory { StateServerFactory.getStateServer().getNode().apply(task); // 提交数据 } catch (CodecException e) { String errorMsg = "Fail to encode TaskState"; - log.error("{}:{}",errorMsg, e); + log.debug("{}:{}",errorMsg, e); closure.failure(errorMsg, PeerId.emptyPeer()); closure.run(new Status(RaftError.EINTERNAL, errorMsg)); } @@ -252,7 +254,7 @@ public class StateServerFactory { public void run(Status status, long index, byte[] reqCtx) { if(status.isOk()){ - log.error("readIndex {}", fsmState); + log.debug("readIndex {}", fsmState); closure.success(fsmState); closure.run(Status.OK()); return; @@ -260,7 +262,7 @@ public class StateServerFactory { readIndexExecutor.execute(() -> { if(isLeader()){ - log.error("Fail to get value with 'ReadIndex': {}, try to applying to the state machine.", status); + log.debug("Fail to get value with 'ReadIndex': {}, try to applying to the state machine.", status); applyState(fsmState, closure); }else { handlerNotLeaderError(closure); @@ -310,6 +312,8 @@ public class StateServerFactory { } } + + public static void main(String[] args) throws InterruptedException, RemotingException { var rpcClient = new BoltRaftRpcFactory().createRpcClient(); rpcClient.init(new CliOptions()); diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml index 269a553..fe5dcb1 100644 --- a/src/main/resources/logback.xml +++ b/src/main/resources/logback.xml @@ -6,8 +6,19 @@ %d{yyyyMMdd HH:mm:ss.SSS} %level %thread\(%file:%line\): %msg%n + + DEBUG + ACCEPT + DENY + - + + + + + + + \ No newline at end of file From 9e6159ee16d3f7d7fd114d51d81bc200b9272415 Mon Sep 17 00:00:00 2001 From: huangsimin Date: Tue, 19 Jul 2022 12:06:03 +0800 Subject: [PATCH 11/35] =?UTF-8?q?=E5=AE=8C=E6=88=90=E5=90=8C=E6=AD=A5?= =?UTF-8?q?=E6=9B=B4=E6=96=B0=E7=9A=84api?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../yuandian/dataflow/controller/TaskLog.java | 17 +- .../dataflow/statemachine/StateMachine.java | 27 ++-- .../statemachine/StateServerFactory.java | 150 +++++++++++------- .../dataflow/statemachine/SyncClosure.java | 4 +- .../statemachine/state/WorkerState.java | 4 + 5 files changed, 123 insertions(+), 79 deletions(-) diff --git a/src/main/java/com/yuandian/dataflow/controller/TaskLog.java b/src/main/java/com/yuandian/dataflow/controller/TaskLog.java index 01ae05a..216a65c 100644 --- a/src/main/java/com/yuandian/dataflow/controller/TaskLog.java +++ b/src/main/java/com/yuandian/dataflow/controller/TaskLog.java @@ -25,25 +25,18 @@ public class TaskLog { @GetMapping(path = "/test") public ResponseEntity Processing() throws InterruptedException, RemotingException { - + + var ws = new WorkerState(new PeerId()); + StateServerFactory.getStateServer().updateFsmWorkerState(ws); + final Response response = new Response(); StateServerFactory.getStateServer().useFsmState((fsmState)->{ log.debug(fsmState.toString() ); + response.Message = fsmState.toString(); return null; }); - // state.getWorker().setPeerId(StateServerFactory.getStateServer().getNode().getNodeId().getPeerId()); - // state.getWorker().setTaskQueueSize(1); - - StateServerFactory.getStateServer().updateFsmState((fsmState)->{ - log.debug(fsmState.toString() ); - fsmState.getWorkers().put(new PeerId(), new WorkerState()); - return fsmState; - }); - - final Response response = new Response(); response.Code = HttpStatus.OK; - response.Message = "ok"; return new ResponseEntity(response, HttpStatus.OK); } diff --git a/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java b/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java index 8019d81..ede6e55 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java @@ -167,16 +167,19 @@ public class StateMachine extends StateMachineAdapter { this.leaderTerm.set(term); super.onLeaderStart(term); - var ss = StateServerFactory.getStateServer(); - var ws = new WorkerState(); - ws.setPeerId(ss.getCluster().getServerId()); - final SyncClosure closure = new SyncClosure< State>() { - @Override - public void run(Status status) { - log.debug("leader set WorkerState {} ", status); - } - }; - ss.applyWorkerState(ws, closure); + try { + updateState((state)->{ + var ws = new WorkerState(StateServerFactory.getStateServer().getCluster().getServerId()); + state.getWorkers().put(ws.peerId, ws); + return state; + }); + } catch (InterruptedException e) { + e.printStackTrace(); + } catch (RemotingException e) { + e.printStackTrace(); + } + + return; } @@ -184,10 +187,10 @@ public class StateMachine extends StateMachineAdapter { public void onStartFollowing(LeaderChangeContext ctx) { super.onStartFollowing(ctx); + try { var ss = StateServerFactory.getStateServer(); - var ws = new WorkerState(); - ws.setPeerId(ss.getCluster().getServerId()); + var ws = new WorkerState(ss.getCluster().getServerId()); var request = new RequestCondition(); request.setWorkerState(ws); log.debug("my: {} leader id {}",ss.getCluster().getServerId(), ss.getNode().getLeaderId()); diff --git a/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java b/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java index 93173ea..3e6338c 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java @@ -33,6 +33,7 @@ import com.alipay.sofa.jraft.rpc.impl.BoltRaftRpcFactory; import com.alipay.sofa.jraft.util.BytesUtil; import com.alipay.sofa.jraft.util.Endpoint; import com.alipay.sofa.jraft.util.ThreadPoolUtil; +import com.yuandian.dataflow.statemachine.rpc.RequestCondition; import com.yuandian.dataflow.statemachine.rpc.RequestState; import com.yuandian.dataflow.statemachine.rpc.ResponseSM; import com.yuandian.dataflow.statemachine.rpc.SyncConditionProcessor; @@ -126,9 +127,12 @@ public class StateServerFactory { } + /** + * 同步 可以使用follow使用, 但是可能延迟于leader. 只读 + * @param dofunc + */ public void useFsmState(Function dofunc) { - - + SyncClosure closure = new SyncClosure() { @Override public void run(Status status) { @@ -136,17 +140,18 @@ public class StateServerFactory { } }; - + getNode().readIndex(BytesUtil.EMPTY_BYTES, new ReadIndexClosure() { @Override public void run(Status status, long index, byte[] reqCtx) { getFsm().useState((fsmState)->{ if(status.isOk()){ - log.debug("readIndex {}", fsmState); closure.success(fsmState); closure.run(Status.OK()); - dofunc.apply(fsmState); + synchronized(dofunc) { + dofunc.apply(fsmState); + } return null; } @@ -160,22 +165,99 @@ public class StateServerFactory { }); return null; }); - - + + synchronized(dofunc) { + log.debug("dofunc notify"); + dofunc.notify(); + } } }); - - - + + try { + synchronized(dofunc) { + log.debug("dofunc wait"); + dofunc.wait(5000); + log.debug("dofunc unwait"); + } + } catch (InterruptedException e) { + e.printStackTrace(); + } return ; } - public void updateFsmState(Function dofunc) throws InterruptedException, RemotingException { - this.getFsm().updateState(dofunc); - } + /** + * 同步更新 WorkerState + * @param dofunc + * @throws InterruptedException + * @throws RemotingException + */ + public void updateFsmWorkerState(WorkerState ws) throws InterruptedException, RemotingException { - + if(isLeader()) { + var closure = new SyncClosure() { + @Override + public void run(Status status) { + log.debug("leader {}", status); + this.synclock.notify(); + } + }; + StateServerFactory.getStateServer().applyWorkerState(ws, closure); + closure.synclock.wait(5000); + return; + } + + try { + var ss = StateServerFactory.getStateServer(); + var request = new RequestCondition(); + request.setWorkerState(ws); + log.debug("my: {} leader id {}",ss.getCluster().getServerId(), ss.getNode().getLeaderId()); + ResponseSM resp; + resp = (ResponseSM)StateServerFactory.getStateServer().getRpcClient().invokeSync(ss.getNode().getLeaderId().getEndpoint(), request, 5000); + if(resp == null) { + log.error("{} set WorkerState is error", resp); + } + log.debug("follow is {}", resp); + return; + } catch (InterruptedException | RemotingException e) { + e.printStackTrace(); + } + } + + /** + * 同步更新整个State + * @param s + * @throws InterruptedException + * @throws RemotingException + */ + public void updateFsmState(State s) throws InterruptedException, RemotingException { + if(isLeader()) { + var closure = new SyncClosure() { + @Override + public void run(Status status) { + log.debug("leader {}", status); + synclock.notify(); + } + }; + StateServerFactory.getStateServer().applyState(s, closure); + closure.synclock.wait(5000); + + return; + } + + var ss = StateServerFactory.getStateServer(); + var request = new RequestState(); + request.setState(s); + log.debug("my: {} leader id {}",ss.getCluster().getServerId(), ss.getNode().getLeaderId()); + ResponseSM resp; + resp = (ResponseSM)StateServerFactory.getStateServer().getRpcClient().invokeSync(ss.getNode().getLeaderId().getEndpoint(), request, 5000); + if(resp == null) { + log.error("{} set State is error", resp); + } + log.debug("follow is {}", resp); + return; + + } public void applyState(State state, SyncClosure closure) { @@ -235,46 +317,6 @@ public class StateServerFactory { - } - - public void readIndexState(final boolean readOnlySafe, final SyncClosure closure) { - - - useFsmState((fsmState)->{ - closure.setValue(fsmState); - if(!readOnlySafe){ - - closure.success(fsmState); - closure.run(Status.OK()); - return null; - } - - getNode().readIndex(BytesUtil.EMPTY_BYTES, new ReadIndexClosure() { - @Override - public void run(Status status, long index, byte[] reqCtx) { - - if(status.isOk()){ - log.debug("readIndex {}", fsmState); - closure.success(fsmState); - closure.run(Status.OK()); - return; - } - - readIndexExecutor.execute(() -> { - if(isLeader()){ - log.debug("Fail to get value with 'ReadIndex': {}, try to applying to the state machine.", status); - applyState(fsmState, closure); - }else { - handlerNotLeaderError(closure); - } - }); - } - }); - - return null; - }); - - } public ResponseSM redirect() { diff --git a/src/main/java/com/yuandian/dataflow/statemachine/SyncClosure.java b/src/main/java/com/yuandian/dataflow/statemachine/SyncClosure.java index f19f155..40ba6d9 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/SyncClosure.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/SyncClosure.java @@ -25,7 +25,9 @@ public abstract class SyncClosure implements Closure { // 代表任务状态 private T value; - + public Object synclock = new Object(); + + public SyncClosure() { } diff --git a/src/main/java/com/yuandian/dataflow/statemachine/state/WorkerState.java b/src/main/java/com/yuandian/dataflow/statemachine/state/WorkerState.java index bc7e058..2d46d94 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/state/WorkerState.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/state/WorkerState.java @@ -29,4 +29,8 @@ public class WorkerState implements Serializable { // 节点的对应peerID public PeerId peerId; public long taskQueueSize; + + public WorkerState(PeerId pid) { + this.peerId = pid; + } } From d7cd8ed7580b2c985b3a1c78f2cb735e4be09503 Mon Sep 17 00:00:00 2001 From: huangsimin Date: Tue, 19 Jul 2022 18:26:56 +0800 Subject: [PATCH 12/35] =?UTF-8?q?TODO:=20=E5=90=84=E4=B8=AA=E8=8A=82?= =?UTF-8?q?=E7=82=B9=E7=9A=84=E9=94=80=E6=AF=81=E5=85=B3=E7=B3=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/yuandian/dataflow/Server.java | 7 +- .../yuandian/dataflow/controller/TaskLog.java | 28 +++++--- .../dataflow/statemachine/StateMachine.java | 45 ++++++++----- .../statemachine/StateServerFactory.java | 66 +++++++++++++++++-- src/main/resources/logback.xml | 4 +- .../java/com/yuandian/dataflow/AppTest.java | 1 + start.sh | 3 +- 7 files changed, 115 insertions(+), 39 deletions(-) diff --git a/src/main/java/com/yuandian/dataflow/Server.java b/src/main/java/com/yuandian/dataflow/Server.java index 89ce643..4a49a81 100644 --- a/src/main/java/com/yuandian/dataflow/Server.java +++ b/src/main/java/com/yuandian/dataflow/Server.java @@ -38,8 +38,7 @@ public class Server { public static void main(String[] args) throws Exception { - - + String[] peers = new String[]{"localhost:4440","localhost:4441","localhost:4442"}; String[] sprPeers = new String[]{"3440","3441","3442"}; @@ -54,12 +53,10 @@ public class Server { Configuration conf = JRaftUtils.getConfiguration("localhost:4440,localhost:4441,localhost:4442"); - StateServerFactory.InitStateServer(peeridstr, conf); + StateServerFactory.initStateServer(peeridstr, conf); System.setProperty("server.port", sprPort); var app = SpringApplication.run(Server.class, args); app.start(); - - } } diff --git a/src/main/java/com/yuandian/dataflow/controller/TaskLog.java b/src/main/java/com/yuandian/dataflow/controller/TaskLog.java index 216a65c..4a48904 100644 --- a/src/main/java/com/yuandian/dataflow/controller/TaskLog.java +++ b/src/main/java/com/yuandian/dataflow/controller/TaskLog.java @@ -24,26 +24,36 @@ public class TaskLog { @GetMapping(path = "/test") public ResponseEntity Processing() throws InterruptedException, RemotingException { - - - var ws = new WorkerState(new PeerId()); - StateServerFactory.getStateServer().updateFsmWorkerState(ws); - final Response response = new Response(); + // var ws = new WorkerState(new PeerId()); + // StateServerFactory.getStateServer().updateFsmWorkerState(ws); + Response response = new Response(); StateServerFactory.getStateServer().useFsmState((fsmState)->{ log.debug(fsmState.toString() ); + log.debug( StateServerFactory.getNode().getLeaderId().toString() ); response.Message = fsmState.toString(); return null; }); - - response.Code = HttpStatus.OK; return new ResponseEntity(response, HttpStatus.OK); } @GetMapping(path = "/test2") - public ResponseEntity MongodbTest(@RequestParam int status) { - + public ResponseEntity MongodbTest() { Response response = new Response(); + StateServerFactory.getStateServer().useFsmStateAsync((fsmState)->{ + log.debug("{} {}", fsmState.toString()); + // response.Message = fsmState.toString(); + return null; + }); + + return new ResponseEntity(response, HttpStatus.OK); + } + + + @GetMapping(path = "/test3") + public ResponseEntity RemoveLeader() { + Response response = new Response(); + StateServerFactory.getNode().shutdown(); return new ResponseEntity(response, HttpStatus.OK); } } diff --git a/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java b/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java index ede6e55..4d292da 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java @@ -111,7 +111,7 @@ public class StateMachine extends StateMachineAdapter { @Override - // @SuppressWarnings("unchecked") + @SuppressWarnings("unchecked") public void onApply(final Iterator iter) { while (iter.hasNext()) { @@ -131,7 +131,7 @@ public class StateMachine extends StateMachineAdapter { final ByteBuffer data = iter.getData(); try { synchronized(state) { - state = SerializerManager.getSerializer(SerializerManager.Hessian2).deserialize( + state = SerializerManager.getSerializer(SerializerManager.Hessian2).deserialize( data.array(), State.class.getName()); log.info("SyncDataClosure(null) taskState:{} leaderTerm:{}", state, this.leaderTerm); } @@ -153,20 +153,17 @@ public class StateMachine extends StateMachineAdapter { @Override public void onError(final RaftException e) { - log.debug("Raft error: {}", e, e); + log.error("Raft error: {}", e, e); } @Override public boolean onSnapshotLoad(final SnapshotReader reader) { - return true; } @Override public void onLeaderStart(final long term) { this.leaderTerm.set(term); - super.onLeaderStart(term); - try { updateState((state)->{ var ws = new WorkerState(StateServerFactory.getStateServer().getCluster().getServerId()); @@ -178,15 +175,31 @@ public class StateMachine extends StateMachineAdapter { } catch (RemotingException e) { e.printStackTrace(); } - - - return; + super.onLeaderStart(term); } + @Override + public void onLeaderStop(final Status status) { + this.leaderTerm.set(-1); + super.onLeaderStop(status); + + try { + updateState((state)->{ + state.getWorkers().remove( StateServerFactory.getServerId() ); + return state; + }); + } catch (InterruptedException e) { + e.printStackTrace(); + } catch (RemotingException e) { + e.printStackTrace(); + } + } + + @Override public void onStartFollowing(LeaderChangeContext ctx) { - super.onStartFollowing(ctx); + try { var ss = StateServerFactory.getStateServer(); @@ -204,15 +217,15 @@ public class StateMachine extends StateMachineAdapter { } catch (InterruptedException | RemotingException e) { e.printStackTrace(); } - + + super.onStartFollowing(ctx); } - @Override - public void onLeaderStop(final Status status) { - this.leaderTerm.set(-1); - super.onLeaderStop(status); - } + + + + public static void main(String[] args) throws InterruptedException, RemotingException { } diff --git a/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java b/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java index 3e6338c..5f01dc3 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java @@ -58,13 +58,28 @@ public class StateServerFactory { private static StateServer ss; - public static void InitStateServer(String peerstr, Configuration conf) throws Exception { + public static void initStateServer(String peerstr, Configuration conf) throws Exception { if(ss != null) { throw new Exception("重复初始化 InitStateServer"); } ss = new StateServerFactory.StateServer(peerstr, conf); log.debug("init peerid {}", ss.node.getNodeId().getPeerId()); + } + public static PeerId getServerId() { + return ss.getCluster().getServerId(); + } + + public static Node getNode() { + return ss.getNode() ; + } + + public static RpcClient getRpcClient() { + return ss.getRpcClient(); + } + + public static RaftGroupService getCluster() { + return ss.getCluster(); } // 获取状态服务的对象 @@ -136,11 +151,11 @@ public class StateServerFactory { SyncClosure closure = new SyncClosure() { @Override public void run(Status status) { - + } }; - + getNode().readIndex(BytesUtil.EMPTY_BYTES, new ReadIndexClosure() { @Override public void run(Status status, long index, byte[] reqCtx) { @@ -167,7 +182,7 @@ public class StateServerFactory { }); synchronized(dofunc) { - log.debug("dofunc notify"); + // log.debug("dofunc notify {}", getNode()); dofunc.notify(); } } @@ -175,9 +190,9 @@ public class StateServerFactory { try { synchronized(dofunc) { - log.debug("dofunc wait"); + // log.debug("dofunc wait"); dofunc.wait(5000); - log.debug("dofunc unwait"); + // log.debug("dofunc unwait"); } } catch (InterruptedException e) { e.printStackTrace(); @@ -186,6 +201,43 @@ public class StateServerFactory { } + public void useFsmStateAsync(Function dofunc) { + + SyncClosure closure = new SyncClosure() { + @Override + public void run(Status status) { + dofunc.apply(this.getValue()); + } + }; + + getNode().readIndex(BytesUtil.EMPTY_BYTES, new ReadIndexClosure() { + @Override + public void run(Status status, long index, byte[] reqCtx) { + + getFsm().useState((fsmState)->{ + if(status.isOk()){ + closure.setValue(fsmState); + closure.success(fsmState); + closure.run(Status.OK()); + return null; + } + + readIndexExecutor.execute(() -> { + if(isLeader()){ + log.info("Fail to get value with 'ReadIndex': {}, try to applying to the state machine.", getNode()); + applyState(fsmState, closure); + }else { + handlerNotLeaderError(closure); + } + }); + return null; + }); + } + }); + + return ; + } + /** * 同步更新 WorkerState * @param dofunc @@ -194,6 +246,7 @@ public class StateServerFactory { */ public void updateFsmWorkerState(WorkerState ws) throws InterruptedException, RemotingException { + // leader就直接提交 if(isLeader()) { var closure = new SyncClosure() { @Override @@ -208,6 +261,7 @@ public class StateServerFactory { } try { + // 非leader就 rpc请求 var ss = StateServerFactory.getStateServer(); var request = new RequestCondition(); request.setWorkerState(ws); diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml index fe5dcb1..4b84335 100644 --- a/src/main/resources/logback.xml +++ b/src/main/resources/logback.xml @@ -6,11 +6,11 @@ %d{yyyyMMdd HH:mm:ss.SSS} %level %thread\(%file:%line\): %msg%n - + diff --git a/src/test/java/com/yuandian/dataflow/AppTest.java b/src/test/java/com/yuandian/dataflow/AppTest.java index 14a9c6b..0d6d478 100644 --- a/src/test/java/com/yuandian/dataflow/AppTest.java +++ b/src/test/java/com/yuandian/dataflow/AppTest.java @@ -41,6 +41,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j public class AppTest { + @FunctionalInterface public interface FuncReturn { public float Execute(); diff --git a/start.sh b/start.sh index 42bed0d..748c796 100755 --- a/start.sh +++ b/start.sh @@ -1,6 +1,6 @@ screen -S raft-0 -X quit screen -S raft-1 -X quit -screen -S raft-2 -X quit +screen -S raft-2 -X quit sleep 1 @@ -8,6 +8,7 @@ screen -dmS raft-0 -L java -jar target/dataflow-1.0.0-SNAPSHOT.jar 0 screen -dmS raft-1 -L java -jar target/dataflow-1.0.0-SNAPSHOT.jar 1 screen -dmS raft-2 -L java -jar target/dataflow-1.0.0-SNAPSHOT.jar 2 +sleep 1 screen -S raft-0 -X logfile flush 0 screen -S raft-1 -X logfile flush 0 From 62dd411739e7533295eeeefe4804864648573e4b Mon Sep 17 00:00:00 2001 From: huangsimin Date: Wed, 20 Jul 2022 01:47:49 +0800 Subject: [PATCH 13/35] =?UTF-8?q?TODO:=20=E6=96=B0=E6=A8=A1=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/yuandian/dataflow/Server.java | 5 +- .../yuandian/dataflow/controller/TaskLog.java | 7 +- .../dataflow/statemachine/StateMachine.java | 54 +++++++++++--- .../statemachine/StateServerFactory.java | 74 +++++++++++++++++-- src/main/resources/logback.xml | 8 +- 5 files changed, 124 insertions(+), 24 deletions(-) diff --git a/src/main/java/com/yuandian/dataflow/Server.java b/src/main/java/com/yuandian/dataflow/Server.java index 4a49a81..6258617 100644 --- a/src/main/java/com/yuandian/dataflow/Server.java +++ b/src/main/java/com/yuandian/dataflow/Server.java @@ -8,6 +8,7 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.SpringBootConfiguration; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration; +import org.springframework.context.ConfigurableApplicationContext; import com.alipay.remoting.serialization.SerializerManager; import com.alipay.sofa.jraft.JRaftUtils; @@ -49,14 +50,14 @@ public class Server { // var peeridstr = peers[2]; // var sprPort = sprPeers[2]; - + Configuration conf = JRaftUtils.getConfiguration("localhost:4440,localhost:4441,localhost:4442"); StateServerFactory.initStateServer(peeridstr, conf); System.setProperty("server.port", sprPort); - var app = SpringApplication.run(Server.class, args); + ConfigurableApplicationContext app = SpringApplication.run(Server.class, args); app.start(); } } diff --git a/src/main/java/com/yuandian/dataflow/controller/TaskLog.java b/src/main/java/com/yuandian/dataflow/controller/TaskLog.java index 4a48904..4589103 100644 --- a/src/main/java/com/yuandian/dataflow/controller/TaskLog.java +++ b/src/main/java/com/yuandian/dataflow/controller/TaskLog.java @@ -53,7 +53,12 @@ public class TaskLog { @GetMapping(path = "/test3") public ResponseEntity RemoveLeader() { Response response = new Response(); - StateServerFactory.getNode().shutdown(); + try { + StateServerFactory.getNode().shutdown(); + } catch (Exception e) { + e.printStackTrace(); + } + return new ResponseEntity(response, HttpStatus.OK); } } diff --git a/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java b/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java index 4d292da..90eddb4 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java @@ -9,11 +9,14 @@ import java.util.function.Function; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.boot.SpringApplication; + import com.alipay.remoting.exception.CodecException; import com.alipay.remoting.serialization.SerializerManager; import com.alipay.sofa.jraft.Closure; import com.alipay.sofa.jraft.Iterator; import com.alipay.sofa.jraft.Status; +import com.alipay.sofa.jraft.conf.Configuration; import com.alipay.sofa.jraft.core.StateMachineAdapter; import com.alipay.sofa.jraft.entity.LeaderChangeContext; import com.alipay.sofa.jraft.entity.PeerId; @@ -153,7 +156,7 @@ public class StateMachine extends StateMachineAdapter { @Override public void onError(final RaftException e) { - log.error("Raft error: {}", e, e); + log.debug("Raft error: {}", e, e); } @Override @@ -163,6 +166,7 @@ public class StateMachine extends StateMachineAdapter { @Override public void onLeaderStart(final long term) { + log.debug("onLeaderStart {}", StateServerFactory.getCluster().getServerId()); this.leaderTerm.set(term); try { updateState((state)->{ @@ -180,6 +184,7 @@ public class StateMachine extends StateMachineAdapter { @Override public void onLeaderStop(final Status status) { + log.debug("onLeaderStop {}", StateServerFactory.getCluster().getServerId()); this.leaderTerm.set(-1); super.onLeaderStop(status); @@ -188,19 +193,39 @@ public class StateMachine extends StateMachineAdapter { state.getWorkers().remove( StateServerFactory.getServerId() ); return state; }); + } catch (InterruptedException e) { e.printStackTrace(); } catch (RemotingException e) { e.printStackTrace(); } } + + + + @Override + public void onShutdown() { + log.debug("onShutdown"); + try { + updateState((state)->{ + state.getWorkers().remove( StateServerFactory.getServerId() ); + return state; + }); + + } catch (InterruptedException e) { + e.printStackTrace(); + } catch (RemotingException e) { + e.printStackTrace(); + } + + super.onShutdown(); + + } @Override public void onStartFollowing(LeaderChangeContext ctx) { - - - + log.debug("{} {}", ctx, StateServerFactory.getCluster().getServerId()); try { var ss = StateServerFactory.getStateServer(); var ws = new WorkerState(ss.getCluster().getServerId()); @@ -210,7 +235,7 @@ public class StateMachine extends StateMachineAdapter { ResponseSM resp; resp = (ResponseSM)StateServerFactory.getStateServer().getRpcClient().invokeSync(ss.getNode().getLeaderId().getEndpoint(), request, 5000); if(resp == null) { - log.error("{} set WorkerState is error", resp); + log.debug("{} set WorkerState is error", resp); } log.debug("WorkerState is {}", resp); return; @@ -221,13 +246,22 @@ public class StateMachine extends StateMachineAdapter { super.onStartFollowing(ctx); } - + @Override + public void onConfigurationCommitted(Configuration conf) { + // TODO Auto-generated method stub + super.onConfigurationCommitted(conf); + } + + @Override + public void onStopFollowing(LeaderChangeContext ctx) { + log.debug("{} {}",ctx, StateServerFactory.getCluster().getServerId()); + super.onStopFollowing(ctx); + } - - public static void main(String[] args) throws InterruptedException, RemotingException { - - } + + + } diff --git a/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java b/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java index 5f01dc3..5fc2b2e 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java @@ -13,6 +13,8 @@ import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.function.Function; +import org.springframework.context.ConfigurableApplicationContext; + import com.alipay.remoting.NamedThreadFactory; import com.alipay.remoting.exception.CodecException; import com.alipay.remoting.serialization.SerializerManager; @@ -28,6 +30,7 @@ import com.alipay.sofa.jraft.error.RaftError; import com.alipay.sofa.jraft.error.RemotingException; import com.alipay.sofa.jraft.option.CliOptions; import com.alipay.sofa.jraft.option.NodeOptions; +import com.alipay.sofa.jraft.rpc.InvokeCallback; import com.alipay.sofa.jraft.rpc.RpcClient; import com.alipay.sofa.jraft.rpc.impl.BoltRaftRpcFactory; import com.alipay.sofa.jraft.util.BytesUtil; @@ -57,6 +60,7 @@ import lombok.extern.slf4j.Slf4j; public class StateServerFactory { private static StateServer ss; + private static ConfigurableApplicationContext appCxt; public static void initStateServer(String peerstr, Configuration conf) throws Exception { if(ss != null) { @@ -66,6 +70,15 @@ public class StateServerFactory { log.debug("init peerid {}", ss.node.getNodeId().getPeerId()); } + public static void setAppCxt(ConfigurableApplicationContext cxt) { + appCxt = cxt; + } + + public static ConfigurableApplicationContext getAppCxt() { + return appCxt; + } + + public static PeerId getServerId() { return ss.getCluster().getServerId(); } @@ -132,6 +145,8 @@ public class StateServerFactory { cluster.getRpcServer().registerProcessor(new SyncStateProcessor()); cluster.getRpcServer().registerProcessor(new SyncConditionProcessor()); node = cluster.start(); + + rpcClient = new BoltRaftRpcFactory().createRpcClient(); rpcClient.init(new CliOptions()); @@ -206,7 +221,9 @@ public class StateServerFactory { SyncClosure closure = new SyncClosure() { @Override public void run(Status status) { - dofunc.apply(this.getValue()); + if(status.isOk()) { + dofunc.apply(this.getValue()); + } } }; @@ -251,7 +268,7 @@ public class StateServerFactory { var closure = new SyncClosure() { @Override public void run(Status status) { - log.debug("leader {}", status); + // log.debug("leader {}", status); this.synclock.notify(); } }; @@ -267,11 +284,11 @@ public class StateServerFactory { request.setWorkerState(ws); log.debug("my: {} leader id {}",ss.getCluster().getServerId(), ss.getNode().getLeaderId()); ResponseSM resp; - resp = (ResponseSM)StateServerFactory.getStateServer().getRpcClient().invokeSync(ss.getNode().getLeaderId().getEndpoint(), request, 5000); + resp = (ResponseSM)StateServerFactory.getRpcClient().invokeSync(StateServerFactory.getNode().getLeaderId().getEndpoint(), request, 5000); if(resp == null) { log.error("{} set WorkerState is error", resp); } - log.debug("follow is {}", resp); + // log.debug("follow is {}", resp); return; } catch (InterruptedException | RemotingException e) { e.printStackTrace(); @@ -289,13 +306,12 @@ public class StateServerFactory { var closure = new SyncClosure() { @Override public void run(Status status) { - log.debug("leader {}", status); + // log.debug("leader update {}", status); synclock.notify(); } }; StateServerFactory.getStateServer().applyState(s, closure); closure.synclock.wait(5000); - return; } @@ -308,7 +324,51 @@ public class StateServerFactory { if(resp == null) { log.error("{} set State is error", resp); } - log.debug("follow is {}", resp); + // log.debug("follow is {}", resp); + return; + + } + + + public void updateFsmStateAsync(State s, Function onCompleted) throws InterruptedException, RemotingException { + if(isLeader()) { + var closure = new SyncClosure() { + @Override + public void run(Status status) { + // log.debug("leader update {}", status); + if(status.isOk()) { + onCompleted.apply(status); + } + } + }; + StateServerFactory.getStateServer().applyState(s, closure); + return; + } + + var ss = StateServerFactory.getStateServer(); + var request = new RequestState(); + request.setState(s); + log.debug("my: {} leader id {}",ss.getCluster().getServerId(), ss.getNode().getLeaderId()); + + StateServerFactory + .getStateServer() + .getRpcClient() + .invokeAsync(ss.getNode().getLeaderId().getEndpoint(), request, new InvokeCallback() { + + @Override + public void complete(Object result, Throwable err) { + if(result != null){ + onCompleted.apply(Status.OK()); + } else { + var status = new Status(10000, "rpc invokeAsync with request: {}", request); + log.debug("{}", status); + onCompleted.apply(status); + } + } + + }, 5000); + + // log.debug("follow is {}", resp); return; } diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml index 4b84335..f3f6718 100644 --- a/src/main/resources/logback.xml +++ b/src/main/resources/logback.xml @@ -14,11 +14,11 @@ - + - + \ No newline at end of file From 182d38595f714a5f7fc5728aef6fc3b9d134e3c1 Mon Sep 17 00:00:00 2001 From: huangsimin Date: Wed, 20 Jul 2022 18:27:00 +0800 Subject: [PATCH 14/35] =?UTF-8?q?=E5=8E=BB=E9=99=A4Spring=20Boot?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- assembly.xml | 44 ++++++++ pom.xml | 103 +++++++++++++++++- .../java/com/yuandian/dataflow/Server.java | 53 ++++----- .../yuandian/dataflow/controller/TaskLog.java | 89 ++++++++------- .../dataflow/grpc/CollectPackets.java | 18 +-- .../com/yuandian/dataflow/projo/Response.java | 4 +- .../dataflow/statemachine/StateMachine.java | 43 ++++---- .../statemachine/StateServerFactory.java | 88 +++------------ .../statemachine/state/StateFactory.java | 78 +++++++++++++ .../statemachine/state/WorkerState.java | 13 ++- src/main/resources/logback.xml | 12 +- .../java/com/yuandian/dataflow/AppTest.java | 2 +- start.sh | 9 +- 13 files changed, 363 insertions(+), 193 deletions(-) create mode 100644 assembly.xml create mode 100644 src/main/java/com/yuandian/dataflow/statemachine/state/StateFactory.java diff --git a/assembly.xml b/assembly.xml new file mode 100644 index 0000000..d0a0aa8 --- /dev/null +++ b/assembly.xml @@ -0,0 +1,44 @@ + + + bin + + dir + tar.gz + + false + + + true + lib + runtime + + + + + unix + 0755 + + bin/** + config/** + + + **/src/** + **/target/** + **/.*/** + + + + + + + lib/ + false + + + lib + + + + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml index 384412f..b6954ae 100644 --- a/pom.xml +++ b/pom.xml @@ -21,8 +21,8 @@ 2.3.0 1.32.3 1.7.36 - 1.3.10 - 2.7.0 + 1.3.11 + 2.7.1 3.12.11 2.1.0 1.30 @@ -47,6 +47,20 @@ ${yuandian.common.config.version} + + + + + ch.qos.logback + logback-classic + 1.2.11 + + + + ch.qos.logback + logback-core + 1.2.11 + org.slf4j @@ -54,6 +68,27 @@ ${slf4j.version} + + + + + + + + + + + + + org.yaml @@ -77,11 +112,11 @@ - + @@ -214,7 +249,7 @@ --> - + + + + org.apache.maven.plugins + maven-jar-plugin + + + + true + com.yuandian.dataflow.Server + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + make-assembly + package + + single + + + + + + assembly.xml + + + + + + + + + org.apache.maven.plugins diff --git a/src/main/java/com/yuandian/dataflow/Server.java b/src/main/java/com/yuandian/dataflow/Server.java index 6258617..86232a4 100644 --- a/src/main/java/com/yuandian/dataflow/Server.java +++ b/src/main/java/com/yuandian/dataflow/Server.java @@ -1,26 +1,10 @@ package com.yuandian.dataflow; -import java.io.File; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.SpringApplication; -import org.springframework.boot.SpringBootConfiguration; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration; -import org.springframework.context.ConfigurableApplicationContext; - -import com.alipay.remoting.serialization.SerializerManager; import com.alipay.sofa.jraft.JRaftUtils; -import com.alipay.sofa.jraft.Node; -import com.alipay.sofa.jraft.RaftGroupService; import com.alipay.sofa.jraft.conf.Configuration; -import com.alipay.sofa.jraft.entity.PeerId; -import com.alipay.sofa.jraft.option.NodeOptions; -import com.yuandian.dataflow.statemachine.SyncClosure; -import com.yuandian.dataflow.statemachine.StateMachine; import com.yuandian.dataflow.statemachine.StateServerFactory; -import com.yuandian.dataflow.statemachine.rpc.SyncStateProcessor; import lombok.var; import lombok.extern.slf4j.Slf4j; @@ -33,31 +17,38 @@ import lombok.extern.slf4j.Slf4j; * */ @Slf4j -@SpringBootApplication(exclude = {MongoAutoConfiguration.class}) -@SpringBootConfiguration public class Server { - - public static void main(String[] args) throws Exception { - + public static String peeridstr; + public static String sprPort; + public static Configuration conf ; + public static void main(String[] args) throws Exception { + String[] peers = new String[]{"localhost:4440","localhost:4441","localhost:4442"}; String[] sprPeers = new String[]{"3440","3441","3442"}; - var peeridstr = peers[ Integer.parseInt(args[0] )]; - var sprPort = sprPeers[Integer.parseInt(args[0] )]; - + var peeridx = Integer.parseInt(args[0]); + var peeridstr = peers[ peeridx ]; + var sprPort = sprPeers[ peeridx ]; // var peeridstr = peers[2]; // var sprPort = sprPeers[2]; - + log.info("{} {}", peeridstr, sprPort); - Configuration conf = JRaftUtils.getConfiguration("localhost:4440,localhost:4441,localhost:4442"); - - StateServerFactory.initStateServer(peeridstr, conf); + conf = JRaftUtils.getConfiguration(String.join(",", peers)); + StateServerFactory.startStateServer(peeridstr, conf); + + // System.setProperty("server.port", sprPort); + // ConfigurableApplicationContext app = SpringApplication.run(Server.class, args); + // StateServerFactory.setAppCxt(app); + // app.addApplicationListener(new SpringReadyEvent()); + // app.start(); - System.setProperty("server.port", sprPort); - ConfigurableApplicationContext app = SpringApplication.run(Server.class, args); - app.start(); } + + + + + } diff --git a/src/main/java/com/yuandian/dataflow/controller/TaskLog.java b/src/main/java/com/yuandian/dataflow/controller/TaskLog.java index 4589103..626217a 100644 --- a/src/main/java/com/yuandian/dataflow/controller/TaskLog.java +++ b/src/main/java/com/yuandian/dataflow/controller/TaskLog.java @@ -1,10 +1,13 @@ package com.yuandian.dataflow.controller; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; -import org.springframework.stereotype.Controller; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RequestParam; +// import org.springframework.http.HttpStatus; +// import org.springframework.http.MediaType; +// import org.springframework.http.ResponseEntity; +// import org.springframework.stereotype.Controller; +// import org.springframework.web.bind.annotation.GetMapping; +// import org.springframework.web.bind.annotation.PostMapping; +// import org.springframework.web.bind.annotation.RequestBody; +// import org.springframework.web.bind.annotation.RequestParam; import com.alipay.sofa.jraft.Status; import com.alipay.sofa.jraft.entity.PeerId; @@ -19,46 +22,52 @@ import lombok.var; import lombok.extern.slf4j.Slf4j; @Slf4j -@Controller +// @Controller public class TaskLog { - @GetMapping(path = "/test") - public ResponseEntity Processing() throws InterruptedException, RemotingException { - // var ws = new WorkerState(new PeerId()); - // StateServerFactory.getStateServer().updateFsmWorkerState(ws); - Response response = new Response(); - StateServerFactory.getStateServer().useFsmState((fsmState)->{ - log.debug(fsmState.toString() ); - log.debug( StateServerFactory.getNode().getLeaderId().toString() ); - response.Message = fsmState.toString(); - return null; - }); - response.Code = HttpStatus.OK; - return new ResponseEntity(response, HttpStatus.OK); - } + // @PostMapping(path = "/test", produces={MediaType.APPLICATION_JSON_VALUE}) + // public ResponseEntity Processing(@RequestBody String Count) throws InterruptedException, RemotingException { + // // var ws = new WorkerState(new PeerId()); + // // StateServerFactory.getStateServer().updateFsmWorkerState(ws); + // log.debug("{}", Count); + // Response response = new Response(); + // synchronized(response) { + // StateServerFactory.getStateServer().useFsmStateAsync((fsmState)->{ + // log.debug("http: {}",StateServerFactory.getServerId() ); + // response.Message = fsmState.toString(); + // response.notify(); + // return null; + // }); + // } + + // response.Code = HttpStatus.OK; + // return new ResponseEntity(response, HttpStatus.OK); + // } - @GetMapping(path = "/test2") - public ResponseEntity MongodbTest() { - Response response = new Response(); - StateServerFactory.getStateServer().useFsmStateAsync((fsmState)->{ - log.debug("{} {}", fsmState.toString()); - // response.Message = fsmState.toString(); - return null; - }); + // @GetMapping(path = "/test2") + // public ResponseEntity MongodbTest() { + // Response response = new Response(); + // StateServerFactory.getStateServer().useFsmStateAsync((fsmState)->{ + // log.debug("{} {}", fsmState.toString()); + // // response.Message = fsmState.toString(); + // return null; + // }); - return new ResponseEntity(response, HttpStatus.OK); - } + // return new ResponseEntity(response, HttpStatus.OK); + // } - @GetMapping(path = "/test3") - public ResponseEntity RemoveLeader() { - Response response = new Response(); - try { - StateServerFactory.getNode().shutdown(); - } catch (Exception e) { - e.printStackTrace(); - } + // @GetMapping(path = "/test3") + // public ResponseEntity RemoveLeader() { + // Response response = new Response(); + // try { + // StateServerFactory.getNode().shutdown(); + // } catch (Exception e) { + // e.printStackTrace(); + // response.Message = e.getMessage(); + // return new ResponseEntity(response, HttpStatus.INTERNAL_SERVER_ERROR); + // } - return new ResponseEntity(response, HttpStatus.OK); - } + // return new ResponseEntity(response, HttpStatus.OK); + // } } diff --git a/src/main/java/com/yuandian/dataflow/grpc/CollectPackets.java b/src/main/java/com/yuandian/dataflow/grpc/CollectPackets.java index 45ce4ea..74b3a8b 100644 --- a/src/main/java/com/yuandian/dataflow/grpc/CollectPackets.java +++ b/src/main/java/com/yuandian/dataflow/grpc/CollectPackets.java @@ -28,8 +28,8 @@ import com.yuandian.dataflow.proto.msgtype.*; import io.grpc.ManagedChannelBuilder; import lombok.var; import lombok.extern.slf4j.Slf4j; -import org.springframework.http.*; -import org.springframework.web.client.RestTemplate; +// import org.springframework.http.*; +// import org.springframework.web.client.RestTemplate; /** * description @@ -121,15 +121,15 @@ public class CollectPackets extends CollectPacketsServerImplBase { var result = p.unpack(BacktrackingFlowOuterClass.BacktrackingFlow.class); System.out.println(result.getClass().toString() + " ,val: " + JsonFormat.printer().print(result)); - RestTemplate client = new RestTemplate(); - HttpHeaders headers = new HttpHeaders(); - headers.setContentType(MediaType.APPLICATION_JSON); + // RestTemplate client = new RestTemplate(); + // HttpHeaders headers = new HttpHeaders(); + // headers.setContentType(MediaType.APPLICATION_JSON); - HttpEntity requestEntity = new HttpEntity<>(JsonFormat.printer().print(result), headers); - String url = "http://localhost:3440/test"; - ResponseEntity response = client.exchange(url, HttpMethod.POST, requestEntity, Response.class); + // HttpEntity requestEntity = new HttpEntity<>(JsonFormat.printer().print(result), headers); + // String url = "http://localhost:3440/test"; + // ResponseEntity response = client.exchange(url, HttpMethod.POST, requestEntity, Response.class); - System.out.println("result:" + response.getBody()); + // System.out.println("result:" + response.getBody()); } catch (InvalidProtocolBufferException e) { e.printStackTrace(); } diff --git a/src/main/java/com/yuandian/dataflow/projo/Response.java b/src/main/java/com/yuandian/dataflow/projo/Response.java index 8936a92..05bfe65 100644 --- a/src/main/java/com/yuandian/dataflow/projo/Response.java +++ b/src/main/java/com/yuandian/dataflow/projo/Response.java @@ -1,12 +1,12 @@ package com.yuandian.dataflow.projo; -import org.springframework.http.HttpStatus; +import org.apache.http.HttpStatus; import com.fasterxml.jackson.annotation.JsonProperty; public class Response { @JsonProperty("code") - public HttpStatus Code; + public org.apache.http.HttpStatus Code; @JsonProperty("message") public String Message; @JsonProperty("data") diff --git a/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java b/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java index 90eddb4..f4e6595 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java @@ -9,7 +9,7 @@ import java.util.function.Function; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.boot.SpringApplication; + import com.alipay.remoting.exception.CodecException; import com.alipay.remoting.serialization.SerializerManager; @@ -34,6 +34,7 @@ import com.yuandian.dataflow.statemachine.rpc.ResponseSM; import com.yuandian.dataflow.statemachine.rpc.RequestCondition; import com.yuandian.dataflow.statemachine.rpc.RequestState; import com.yuandian.dataflow.statemachine.state.State; +import com.yuandian.dataflow.statemachine.state.StateFactory; import com.yuandian.dataflow.statemachine.state.WorkerState; import lombok.var; @@ -76,9 +77,7 @@ public class StateMachine extends StateMachineAdapter { * Returns current value. 读取修改都在这个函数域内进行 */ public void useState(Function dofunc) { - synchronized(this.state) { - dofunc.apply(this.state); - } + dofunc.apply(this.state); } /** @@ -87,7 +86,7 @@ public class StateMachine extends StateMachineAdapter { * @throws InterruptedException */ public void updateState(Function dofunc) throws InterruptedException, RemotingException { - synchronized(this.state) { + var newstate = dofunc.apply(this.state); var ss = StateServerFactory.getStateServer(); if(!isLeader()) { @@ -109,7 +108,7 @@ public class StateMachine extends StateMachineAdapter { colsure.setValue(newstate); StateServerFactory.getStateServer().applyState(newstate, colsure); } - } + } @@ -123,9 +122,9 @@ public class StateMachine extends StateMachineAdapter { // parsing. var closure = (SyncClosure)iter.done(); // 只支持单一个State. 全状态机只支持一种提交 log.info("SyncDataClosure(done) taskState:{} leaderTerm:{} {}",this.state, this.leaderTerm, closure); - synchronized(this.state) { + this.state = closure.getValue(); - } + closure.success(state); closure.run(Status.OK()); @@ -174,6 +173,9 @@ public class StateMachine extends StateMachineAdapter { state.getWorkers().put(ws.peerId, ws); return state; }); + if(!StateFactory.getMasterExecute().isAlive()) { + StateFactory.getMasterExecute().start(); + } } catch (InterruptedException e) { e.printStackTrace(); } catch (RemotingException e) { @@ -188,6 +190,10 @@ public class StateMachine extends StateMachineAdapter { this.leaderTerm.set(-1); super.onLeaderStop(status); + if(StateFactory.getMasterExecute().isAlive()) { + StateFactory.getMasterExecute().interrupt(); + } + try { updateState((state)->{ state.getWorkers().remove( StateServerFactory.getServerId() ); @@ -206,27 +212,19 @@ public class StateMachine extends StateMachineAdapter { @Override public void onShutdown() { log.debug("onShutdown"); - try { - updateState((state)->{ - state.getWorkers().remove( StateServerFactory.getServerId() ); - return state; - }); - - } catch (InterruptedException e) { - e.printStackTrace(); - } catch (RemotingException e) { - e.printStackTrace(); - } - super.onShutdown(); - } @Override public void onStartFollowing(LeaderChangeContext ctx) { - log.debug("{} {}", ctx, StateServerFactory.getCluster().getServerId()); + log.debug("[onStartFollowing] {} {}", ctx, StateServerFactory.getCluster().getServerId()); try { + + if(StateFactory.getMasterExecute().isAlive()) { + StateFactory.getMasterExecute().interrupt(); + } + var ss = StateServerFactory.getStateServer(); var ws = new WorkerState(ss.getCluster().getServerId()); var request = new RequestCondition(); @@ -248,7 +246,6 @@ public class StateMachine extends StateMachineAdapter { @Override public void onConfigurationCommitted(Configuration conf) { - // TODO Auto-generated method stub super.onConfigurationCommitted(conf); } diff --git a/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java b/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java index 5fc2b2e..d7d28ab 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java @@ -13,7 +13,7 @@ import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.function.Function; -import org.springframework.context.ConfigurableApplicationContext; + import com.alipay.remoting.NamedThreadFactory; import com.alipay.remoting.exception.CodecException; @@ -60,23 +60,20 @@ import lombok.extern.slf4j.Slf4j; public class StateServerFactory { private static StateServer ss; - private static ConfigurableApplicationContext appCxt; - public static void initStateServer(String peerstr, Configuration conf) throws Exception { + private static String myPeerStr; + private static Configuration raftConf; + + public static void startStateServer(String peerstr, Configuration conf) throws Exception { if(ss != null) { throw new Exception("重复初始化 InitStateServer"); } ss = new StateServerFactory.StateServer(peerstr, conf); - log.debug("init peerid {}", ss.node.getNodeId().getPeerId()); } - public static void setAppCxt(ConfigurableApplicationContext cxt) { - appCxt = cxt; - } + - public static ConfigurableApplicationContext getAppCxt() { - return appCxt; - } + public static PeerId getServerId() { @@ -114,15 +111,15 @@ public class StateServerFactory { private Executor readIndexExecutor = createReadIndexExecutor(); public StateServer(String addr, Configuration conf) { - String[] peers = new String[]{"localhost:4440","localhost:4441","localhost:4442"}; - String[] sprPeers = new String[]{"3440","3441","3442"}; + // String[] peers = new String[]{"localhost:4440","localhost:4441","localhost:4442"}; + // String[] sprPeers = new String[]{"3440","3441","3442"}; // var peeridstr = peers[Integer.parseInt(serverId)]; // var sprPort = sprPeers[Integer.parseInt(args[0])]; // String groupId = "jraft"; - // Configuration conf = JRaftUtils.getConfiguration("localhost:4440,localhost:4441,localhost:4442"); + // conf = JRaftUtils.getConfiguration("localhost:4440,localhost:4441,localhost:4442"); PeerId serverId = JRaftUtils.getPeerId(addr); int port = serverId.getPort(); @@ -146,76 +143,17 @@ public class StateServerFactory { cluster.getRpcServer().registerProcessor(new SyncConditionProcessor()); node = cluster.start(); - - rpcClient = new BoltRaftRpcFactory().createRpcClient(); rpcClient.init(new CliOptions()); } + + public boolean isLeader() { return this.fsm.isLeader(); } - - /** - * 同步 可以使用follow使用, 但是可能延迟于leader. 只读 - * @param dofunc - */ - public void useFsmState(Function dofunc) { - - SyncClosure closure = new SyncClosure() { - @Override - public void run(Status status) { - - } - }; - - getNode().readIndex(BytesUtil.EMPTY_BYTES, new ReadIndexClosure() { - @Override - public void run(Status status, long index, byte[] reqCtx) { - - getFsm().useState((fsmState)->{ - if(status.isOk()){ - closure.success(fsmState); - closure.run(Status.OK()); - synchronized(dofunc) { - dofunc.apply(fsmState); - } - return null; - } - - readIndexExecutor.execute(() -> { - if(isLeader()){ - log.debug("Fail to get value with 'ReadIndex': {}, try to applying to the state machine.", getNode()); - applyState(fsmState, closure); - }else { - handlerNotLeaderError(closure); - } - }); - return null; - }); - - synchronized(dofunc) { - // log.debug("dofunc notify {}", getNode()); - dofunc.notify(); - } - } - }); - - try { - synchronized(dofunc) { - // log.debug("dofunc wait"); - dofunc.wait(5000); - // log.debug("dofunc unwait"); - } - } catch (InterruptedException e) { - e.printStackTrace(); - } - return ; - - } - public void useFsmStateAsync(Function dofunc) { SyncClosure closure = new SyncClosure() { @@ -402,7 +340,7 @@ public class StateServerFactory { } - useFsmState((fsmState)->{ + useFsmStateAsync((fsmState)->{ var wmap = fsmState.getWorkers(); var wstate = wmap.get(state.getPeerId()); diff --git a/src/main/java/com/yuandian/dataflow/statemachine/state/StateFactory.java b/src/main/java/com/yuandian/dataflow/statemachine/state/StateFactory.java new file mode 100644 index 0000000..6f49539 --- /dev/null +++ b/src/main/java/com/yuandian/dataflow/statemachine/state/StateFactory.java @@ -0,0 +1,78 @@ +/** + * description + * + * @author eson + *2022年7月20日-10:00:05 + */ +package com.yuandian.dataflow.statemachine.state; + + +import com.alibaba.nacos.api.cmdb.pojo.Entity; +import com.yuandian.dataflow.statemachine.StateServerFactory; + +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import lombok.var; +import lombok.extern.slf4j.Slf4j; + +/** + * description + * + * @author eson + *2022年7月20日-10:00:05 + */ +@Slf4j +@Getter +@Setter +@ToString +public class StateFactory { + + public static Thread masterExecute = new Thread(new Runnable() { + @Override + public void run() { + try { + while(true) { + log.debug("master execute {}", StateServerFactory.getServerId()); + var alivePeers = StateServerFactory.getCluster().getRaftNode().listAlivePeers(); + log.debug("master execute {}", StateServerFactory.getCluster().getRaftNode().listAlivePeers()); + if(alivePeers != null) { + synchronized(alivePeers) { + StateServerFactory.getStateServer().useFsmStateAsync((state)->{ + alivePeers.forEach((peer)->{ + var ws = state.getWorkers().get(peer); + if(ws != null) { + var cap = 10000 - ws.getTaskQueueSize(); + if(cap > 0) { + log.debug("{}", cap); + + + } + } + }); + alivePeers.notify(); + return null; + }); + } + + } + + Thread.sleep(2000); + } + } catch (InterruptedException e) { + e.printStackTrace(); + } + + } + }); + + public static Thread getMasterExecute() { + return masterExecute; + } + + public static void Init() { + + } + + +} diff --git a/src/main/java/com/yuandian/dataflow/statemachine/state/WorkerState.java b/src/main/java/com/yuandian/dataflow/statemachine/state/WorkerState.java index 2d46d94..5e16daf 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/state/WorkerState.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/state/WorkerState.java @@ -7,6 +7,7 @@ package com.yuandian.dataflow.statemachine.state; import java.io.Serializable; +import java.time.Instant; import com.alipay.sofa.jraft.entity.PeerId; @@ -26,9 +27,19 @@ import lombok.ToString; public class WorkerState implements Serializable { private static final long serialVersionUID = -1L; - // 节点的对应peerID + + /** + * 节点的对应peerID + */ public PeerId peerId; + /** + * 任务队列的数量 + */ public long taskQueueSize; + /** + * 更新时间 + */ + public Instant updateAt; public WorkerState(PeerId pid) { this.peerId = pid; diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml index f3f6718..85b4e0c 100644 --- a/src/main/resources/logback.xml +++ b/src/main/resources/logback.xml @@ -10,15 +10,21 @@ DEBUG ACCEPT DENY + + + + ERROR + ACCEPT + DENY --> - + - + \ No newline at end of file diff --git a/src/test/java/com/yuandian/dataflow/AppTest.java b/src/test/java/com/yuandian/dataflow/AppTest.java index 0d6d478..1117839 100644 --- a/src/test/java/com/yuandian/dataflow/AppTest.java +++ b/src/test/java/com/yuandian/dataflow/AppTest.java @@ -21,7 +21,7 @@ import javax.swing.text.AbstractDocument.BranchElement; import org.bson.Document; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; -import org.springframework.expression.spel.ast.FunctionReference; +// import org.springframework.expression.spel.ast.FunctionReference; import com.mongodb.MongoClient; import com.mongodb.client.model.InsertManyOptions; diff --git a/start.sh b/start.sh index 748c796..366b3f1 100755 --- a/start.sh +++ b/start.sh @@ -2,11 +2,14 @@ screen -S raft-0 -X quit screen -S raft-1 -X quit screen -S raft-2 -X quit + +version=1.0.0-SNAPSHOT + sleep 1 -screen -dmS raft-0 -L java -jar target/dataflow-1.0.0-SNAPSHOT.jar 0 -screen -dmS raft-1 -L java -jar target/dataflow-1.0.0-SNAPSHOT.jar 1 -screen -dmS raft-2 -L java -jar target/dataflow-1.0.0-SNAPSHOT.jar 2 +screen -dmS raft-0 -L java -cp target/dataflow-${version}.jar-bin/lib/*jar -jar target/dataflow-${version}.jar 0 +screen -dmS raft-1 -L java -cp target/dataflow-${version}.jar-bin/lib/*jar -jar target/dataflow-${version}.jar 1 +screen -dmS raft-2 -L java -cp target/dataflow-${version}.jar-bin/lib/*jar -jar target/dataflow-${version}.jar 2 sleep 1 From 8d79341e2305b61c8277b9b9eda88d0024ef4cbc Mon Sep 17 00:00:00 2001 From: huangsimin Date: Thu, 21 Jul 2022 01:43:25 +0800 Subject: [PATCH 15/35] =?UTF-8?q?=E5=88=A0=E9=99=A4springboot=E6=89=93?= =?UTF-8?q?=E5=8C=85=E4=BE=9D=E8=B5=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 4 +- assembly.xml | 24 +++++------ pom.xml | 120 ++++++++++++++++----------------------------------- restart.sh | 2 + start.sh | 9 ++-- stop.sh | 1 + 6 files changed, 59 insertions(+), 101 deletions(-) create mode 100755 restart.sh diff --git a/.gitignore b/.gitignore index 674db64..ffc6ed1 100644 --- a/.gitignore +++ b/.gitignore @@ -79,7 +79,7 @@ local.properties .classpath # Annotation Processing -.apt_generated +.apt_generated* .sts4-cache/ @@ -202,4 +202,4 @@ README.html raftdata -screenlog.* \ No newline at end of file +screenlog.* diff --git a/assembly.xml b/assembly.xml index d0a0aa8..e1c5d6e 100644 --- a/assembly.xml +++ b/assembly.xml @@ -13,25 +13,25 @@ runtime + + + + - unix - 0755 - - bin/** - config/** - - - **/src/** - **/target/** - **/.*/** - + 0664 + ${project.build.directory} + + + *.jar + + - lib/ + ${project.directory} false diff --git a/pom.xml b/pom.xml index b6954ae..c0e9190 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ 3.12.11 2.1.0 1.30 - + 1.2.11 1.0.4 @@ -53,13 +53,13 @@ ch.qos.logback logback-classic - 1.2.11 + ${logback.version} ch.qos.logback logback-core - 1.2.11 + ${logback.version} @@ -67,27 +67,6 @@ slf4j-api ${slf4j.version} - - - - - - - - - - - - - @@ -227,66 +206,63 @@ 1.6.2 - - + org.apache.maven.plugins maven-jar-plugin + true + lib/ com.yuandian.dataflow.Server - + + org.apache.maven.plugins + maven-dependency-plugin + + + copy-dependencies + prepare-package + + copy-dependencies + + + false + false + true + ${project.build.directory}/lib + + + + + + + + org.apache.maven.plugins maven-assembly-plugin make-assembly - package + assembly single + false + + + com.yuandian.dataflow.Server + + assembly.xml @@ -294,29 +270,7 @@ - - + diff --git a/restart.sh b/restart.sh new file mode 100755 index 0000000..3bb4f6e --- /dev/null +++ b/restart.sh @@ -0,0 +1,2 @@ +#! /bin/bash +sh stop.sh & rm raftdata/ -rf && mvn -T4 package && truncate -s 0 screenlog.0 && sh start.sh diff --git a/start.sh b/start.sh index 366b3f1..2b4c76e 100755 --- a/start.sh +++ b/start.sh @@ -1,15 +1,16 @@ +#! /bin/bash screen -S raft-0 -X quit screen -S raft-1 -X quit screen -S raft-2 -X quit -version=1.0.0-SNAPSHOT +VERSION=1.0.0-SNAPSHOT sleep 1 -screen -dmS raft-0 -L java -cp target/dataflow-${version}.jar-bin/lib/*jar -jar target/dataflow-${version}.jar 0 -screen -dmS raft-1 -L java -cp target/dataflow-${version}.jar-bin/lib/*jar -jar target/dataflow-${version}.jar 1 -screen -dmS raft-2 -L java -cp target/dataflow-${version}.jar-bin/lib/*jar -jar target/dataflow-${version}.jar 2 +screen -dmS raft-0 -L java -jar target/dataflow-${VERSION}.jar 0 +screen -dmS raft-1 -L java -jar target/dataflow-${VERSION}.jar 1 +screen -dmS raft-2 -L java -jar target/dataflow-${VERSION}.jar 2 sleep 1 diff --git a/stop.sh b/stop.sh index e34e95a..12f7ef1 100755 --- a/stop.sh +++ b/stop.sh @@ -1,3 +1,4 @@ +#! /bin/bash screen -S raft-0 -X quit screen -S raft-1 -X quit screen -S raft-2 -X quit From b24ac05920097fd7047f47094deb319bfe7e4906 Mon Sep 17 00:00:00 2001 From: huangsimin Date: Thu, 21 Jul 2022 15:48:59 +0800 Subject: [PATCH 16/35] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=20raft=E5=A4=84?= =?UTF-8?q?=E7=90=86=E7=9A=84=E6=B3=A8=E8=A7=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 13 +-- .../dataflow/controller/PacketsProcessor.java | 59 ++++++++++++++ .../dataflow/grpc/CollectPackets.java | 2 +- .../master/{Header.java => Master.java} | 2 +- .../dataflow/statemachine/StateMachine.java | 23 +----- .../statemachine/StateServerFactory.java | 25 ++++-- .../statemachine/rpc/RequestCondition.java | 21 ----- .../statemachine/rpc/RequestState.java | 35 -------- .../rpc/SyncConditionProcessor.java | 18 ++++- .../statemachine/rpc/SyncStateProcessor.java | 26 +++++- .../rpc/annotations/ProcessorRaft.java | 24 ++++++ .../statemachine/state/StateFactory.java | 80 +++++++++++++------ src/main/resources/logback.xml | 4 +- 13 files changed, 212 insertions(+), 120 deletions(-) create mode 100644 src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java rename src/main/java/com/yuandian/dataflow/master/{Header.java => Master.java} (99%) delete mode 100644 src/main/java/com/yuandian/dataflow/statemachine/rpc/RequestCondition.java delete mode 100644 src/main/java/com/yuandian/dataflow/statemachine/rpc/RequestState.java create mode 100644 src/main/java/com/yuandian/dataflow/statemachine/rpc/annotations/ProcessorRaft.java diff --git a/pom.xml b/pom.xml index c0e9190..6aa74f6 100644 --- a/pom.xml +++ b/pom.xml @@ -89,8 +89,13 @@ ${mongo.driver.version} - - + + + org.reflections + reflections + 0.10.2 + + From 08fbedecccb6d50c05bcae4a285b4bb8358a0752 Mon Sep 17 00:00:00 2001 From: huangsimin Date: Thu, 21 Jul 2022 15:49:48 +0800 Subject: [PATCH 17/35] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=8D=95=E5=85=83?= =?UTF-8?q?=E6=B5=8B=E8=AF=95=E7=9A=84=E7=B1=BB=E5=9E=8B=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/yuandian/dataflow/statemachine/StateMachineTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/test/java/com/yuandian/dataflow/statemachine/StateMachineTest.java b/src/test/java/com/yuandian/dataflow/statemachine/StateMachineTest.java index 329abfa..565a0f2 100644 --- a/src/test/java/com/yuandian/dataflow/statemachine/StateMachineTest.java +++ b/src/test/java/com/yuandian/dataflow/statemachine/StateMachineTest.java @@ -4,14 +4,13 @@ import java.util.concurrent.Executor; import org.junit.jupiter.api.Test; -import com.alipay.sofa.jraft.entity.PeerId; import com.alipay.sofa.jraft.error.RemotingException; import com.alipay.sofa.jraft.option.CliOptions; import com.alipay.sofa.jraft.rpc.InvokeCallback; import com.alipay.sofa.jraft.rpc.impl.BoltRaftRpcFactory; import com.alipay.sofa.jraft.util.Endpoint; import com.yuandian.dataflow.statemachine.rpc.ResponseSM; -import com.yuandian.dataflow.statemachine.rpc.RequestState; +import com.yuandian.dataflow.statemachine.rpc.SyncStateProcessor.RequestState; import com.yuandian.dataflow.statemachine.state.State; import lombok.var; From 761a8dd664f293b2234a0c694617143dab2e70bd Mon Sep 17 00:00:00 2001 From: huangsimin Date: Thu, 21 Jul 2022 16:32:48 +0800 Subject: [PATCH 18/35] =?UTF-8?q?TODO:=20=E6=9B=B4=E6=96=B0=E7=8A=B6?= =?UTF-8?q?=E6=80=81=E7=9A=84=20=E9=87=8D=E6=96=B0=E8=AE=BE=E8=AE=A1=20?= =?UTF-8?q?=E9=81=BF=E5=85=8D=E8=87=AA=E9=94=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dataflow/controller/PacketsProcessor.java | 1 + .../statemachine/StateServerFactory.java | 4 +- .../statemachine/state/StateFactory.java | 80 +++++++++++-------- src/main/resources/logback.xml | 6 +- 4 files changed, 54 insertions(+), 37 deletions(-) diff --git a/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java b/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java index c8a3162..ff00dc9 100644 --- a/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java +++ b/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java @@ -45,6 +45,7 @@ public class PacketsProcessor implements RpcProcessor> scans = new Reflections("com.yuandian.dataflow").getTypesAnnotatedWith(ProcessorRaft.class); scans.forEach((pRaftClass)->{ diff --git a/src/main/java/com/yuandian/dataflow/statemachine/state/StateFactory.java b/src/main/java/com/yuandian/dataflow/statemachine/state/StateFactory.java index 6d9c165..0c21e5c 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/state/StateFactory.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/state/StateFactory.java @@ -6,13 +6,16 @@ */ package com.yuandian.dataflow.statemachine.state; - +import java.time.Instant; + import com.alibaba.nacos.shaded.io.grpc.netty.shaded.io.netty.handler.codec.http.HttpContentEncoder.Result; +import com.alipay.sofa.jraft.Status; import com.alipay.sofa.jraft.error.RemotingException; import com.google.protobuf.Any; import com.yuandian.dataflow.controller.PacketsProcessor.PacketsRequest; import com.yuandian.dataflow.proto.msgtype.BacktrackingFlowOuterClass; import com.yuandian.dataflow.statemachine.StateServerFactory; +import com.yuandian.dataflow.statemachine.SyncClosure; import lombok.Getter; import lombok.Setter; @@ -43,47 +46,58 @@ public class StateFactory { if (alivePeers != null) { var ss = StateServerFactory.getStateServer(); StateServerFactory.getStateServer().useFsmStateAsync((state) -> { - synchronized(alivePeers){ - alivePeers.forEach((peer) -> { - var ws = state.getWorkers().get(peer); - if (ws != null) { - var cap = 100 - ws.getTaskQueueSize(); - if (cap > 0) { - log.debug("{}", cap); - var request = new PacketsRequest(); - for(int i = 0; i < cap ; i++ ) { - var p = Any.pack( - BacktrackingFlowOuterClass.BacktrackingFlow.newBuilder() - .setTableId(10086) - .build() - ); + synchronized (alivePeers) { + alivePeers.forEach((peer) -> { + WorkerState ws = state.getWorkers().get(peer); + if (ws != null) { + var cap = 100 - ws.getTaskQueueSize(); + if (cap > 0) { + log.debug("{}", cap); + var request = new PacketsRequest(); + for (int i = 0; i < cap; i++) { + var p = Any.pack( + BacktrackingFlowOuterClass.BacktrackingFlow.newBuilder() + .setTableId(10086) + .build()); request.getPackets().add(p); - } - - try { - var result = ss.getRpcClient().invokeSync(ws.getPeerId().getEndpoint(), request, 5000); - log.info("{}", result); - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch (RemotingException e) { - // TODO Auto-generated catch block - e.printStackTrace(); + } + + try { + var result = ss.getRpcClient().invokeSync(ws.getPeerId().getEndpoint(), + request, 5000); + log.info("{}", result); + ws.setUpdateAt( Instant.now() ); + ws.setTaskQueueSize(ws.getTaskQueueSize() - cap); + + + + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (RemotingException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } } } - } - alivePeers.notifyAll(); - }); - } - + + ss.applyState(state, new SyncClosure() { + public void run(Status status) { + log.debug("{}", status); + }; + } ); + alivePeers.notifyAll(); + }); + } + return null; }); - synchronized(alivePeers){ + synchronized (alivePeers) { alivePeers.wait(5000); } - + } Thread.sleep(2000); diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml index 77df777..e67372b 100644 --- a/src/main/resources/logback.xml +++ b/src/main/resources/logback.xml @@ -20,11 +20,11 @@ - - - + + \ No newline at end of file From 3f8e3d5ad7f5ac91704067b996b3a1e63612632f Mon Sep 17 00:00:00 2001 From: huangsimin Date: Thu, 21 Jul 2022 16:44:39 +0800 Subject: [PATCH 19/35] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=B3=A8=E9=87=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dataflow/statemachine/rpc/annotations/ProcessorRaft.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/yuandian/dataflow/statemachine/rpc/annotations/ProcessorRaft.java b/src/main/java/com/yuandian/dataflow/statemachine/rpc/annotations/ProcessorRaft.java index 71dda04..86426ef 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/rpc/annotations/ProcessorRaft.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/rpc/annotations/ProcessorRaft.java @@ -12,7 +12,7 @@ import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; /** - * raft自动装配 + * raft自定注册到全局唯一raft服务上 * * @author eson *2022年7月21日-14:27:49 From d35d161f4173e36eb36cfca13c8ae004d6960c0f Mon Sep 17 00:00:00 2001 From: huangsimin Date: Thu, 21 Jul 2022 16:45:30 +0800 Subject: [PATCH 20/35] =?UTF-8?q?=E5=88=A0=E9=99=A4=E5=A4=9A=E4=BD=99?= =?UTF-8?q?=E7=9A=84=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../yuandian/dataflow/controller/TaskLog.java | 73 ---------- .../com/yuandian/dataflow/master/Master.java | 137 ------------------ 2 files changed, 210 deletions(-) delete mode 100644 src/main/java/com/yuandian/dataflow/controller/TaskLog.java delete mode 100644 src/main/java/com/yuandian/dataflow/master/Master.java diff --git a/src/main/java/com/yuandian/dataflow/controller/TaskLog.java b/src/main/java/com/yuandian/dataflow/controller/TaskLog.java deleted file mode 100644 index 626217a..0000000 --- a/src/main/java/com/yuandian/dataflow/controller/TaskLog.java +++ /dev/null @@ -1,73 +0,0 @@ -package com.yuandian.dataflow.controller; - -// import org.springframework.http.HttpStatus; -// import org.springframework.http.MediaType; -// import org.springframework.http.ResponseEntity; -// import org.springframework.stereotype.Controller; -// import org.springframework.web.bind.annotation.GetMapping; -// import org.springframework.web.bind.annotation.PostMapping; -// import org.springframework.web.bind.annotation.RequestBody; -// import org.springframework.web.bind.annotation.RequestParam; - -import com.alipay.sofa.jraft.Status; -import com.alipay.sofa.jraft.entity.PeerId; -import com.alipay.sofa.jraft.error.RemotingException; -import com.yuandian.dataflow.projo.Response; -import com.yuandian.dataflow.statemachine.StateServerFactory; -import com.yuandian.dataflow.statemachine.SyncClosure; -import com.yuandian.dataflow.statemachine.state.State; -import com.yuandian.dataflow.statemachine.state.WorkerState; - -import lombok.var; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -// @Controller -public class TaskLog { - - // @PostMapping(path = "/test", produces={MediaType.APPLICATION_JSON_VALUE}) - // public ResponseEntity Processing(@RequestBody String Count) throws InterruptedException, RemotingException { - // // var ws = new WorkerState(new PeerId()); - // // StateServerFactory.getStateServer().updateFsmWorkerState(ws); - // log.debug("{}", Count); - // Response response = new Response(); - // synchronized(response) { - // StateServerFactory.getStateServer().useFsmStateAsync((fsmState)->{ - // log.debug("http: {}",StateServerFactory.getServerId() ); - // response.Message = fsmState.toString(); - // response.notify(); - // return null; - // }); - // } - - // response.Code = HttpStatus.OK; - // return new ResponseEntity(response, HttpStatus.OK); - // } - - // @GetMapping(path = "/test2") - // public ResponseEntity MongodbTest() { - // Response response = new Response(); - // StateServerFactory.getStateServer().useFsmStateAsync((fsmState)->{ - // log.debug("{} {}", fsmState.toString()); - // // response.Message = fsmState.toString(); - // return null; - // }); - - // return new ResponseEntity(response, HttpStatus.OK); - // } - - - // @GetMapping(path = "/test3") - // public ResponseEntity RemoveLeader() { - // Response response = new Response(); - // try { - // StateServerFactory.getNode().shutdown(); - // } catch (Exception e) { - // e.printStackTrace(); - // response.Message = e.getMessage(); - // return new ResponseEntity(response, HttpStatus.INTERNAL_SERVER_ERROR); - // } - - // return new ResponseEntity(response, HttpStatus.OK); - // } -} diff --git a/src/main/java/com/yuandian/dataflow/master/Master.java b/src/main/java/com/yuandian/dataflow/master/Master.java deleted file mode 100644 index 62b0d6f..0000000 --- a/src/main/java/com/yuandian/dataflow/master/Master.java +++ /dev/null @@ -1,137 +0,0 @@ -package com.yuandian.dataflow.master; - -import java.io.ByteArrayInputStream; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.net.InetSocketAddress; -import java.net.Socket; -import java.net.SocketException; -import java.nio.Buffer; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.util.ArrayList; - -// import com.yuandian.dataflow.proto.decode.PacketBase; -// import com.yuandian.dataflow.proto.decode.PacketHeader; -// import com.yuandian.dataflow.proto.decode.utils; - -import io.netty.handler.codec.compression.ZlibDecoder; -import lombok.Cleanup; -import lombok.Getter; -import lombok.Setter; -import lombok.var; -import lombok.extern.slf4j.Slf4j; - -/** - * Header - */ -@Slf4j - -public class Master { - - public static void main(String[] args) throws Exception { - - var addr = new InetSocketAddress("192.168.1.248", 60001); - @Cleanup - var sock = new Socket(); - sock.setReceiveBufferSize(10 * 1024 * 1024);// socket接收缓存 - sock.setSoTimeout(1000 * 30); - // 设置超时 - sock.connect(addr, 10 * 1000); - var in = new DataInputStream(sock.getInputStream()); - var out = new DataOutputStream(sock.getOutputStream()); - // 发送验证字符串 - // out.write("public".getBytes()); - // log.debug("{}", PacketHeader.PacketCode(in)); - // var pheader = new PacketHeader(in); - - // log.debug("{}", pheader); - // var pbase = PacketBase.createPacketBase(pheader); - // log.debug("{}",pbase); - - - // //60010流需要解压 - // byte[] unzipbodydata = null; - // if (pheader.getTableID() == 20) { - // pheader.parseNextHeader_60010(in); - // byte[] zipbodydata = in.readNBytes(pheader.getMsgLen()); - // unzipbodydata = utils.Inflate(zipbodydata); - // } - - // for (int i = 0; i < pheader.getRecCount(); i++) { - // PacketBase dataBean = null; - - // if (pheader.getTableID() >= 22 && pheader.getTableID() <= 24) { - // // 读取具体数据头信息，获取前四个字段值，第四个字段为整条数据的长度 字段长度分别为 4 1 4 4 - // // bodyhead = new byte[13]; - // // readTillLength(bodyhead, 13); - - // var p1 = ByteBuffer.wrap(in.readNBytes(4)).order(ByteOrder.LITTLE_ENDIAN).getInt(); - // var p2 = ByteBuffer.wrap(in.readNBytes(1)).order(ByteOrder.LITTLE_ENDIAN).getInt(); - // var p3 = ByteBuffer.wrap(in.readNBytes(4)).order(ByteOrder.LITTLE_ENDIAN).getInt(); - // var length = ByteBuffer.wrap(in.readNBytes(4)).order(ByteOrder.LITTLE_ENDIAN).getInt(); - - // if (length <= 13) { - // log.debug("[{}:{}][数据总条数:{}][当前解析第{}条][length<13][探针发包有问题]"); - // throw new Exception("数据解析异常"); - // } - - // // 解析数据 - // dataBean = pbase.Parse(pheader, ByteBuffer.wrap(in.readNBytes(length - 13))); - // } else if (pheader.getTableID() == 25) { - - // var nowtype = ByteBuffer.wrap(in.readNBytes(4)).order(ByteOrder.LITTLE_ENDIAN).getInt(); - // var p2 = ByteBuffer.wrap(in.readNBytes(1)).order(ByteOrder.LITTLE_ENDIAN).getInt(); - // var p3 = ByteBuffer.wrap(in.readNBytes(4)).order(ByteOrder.LITTLE_ENDIAN).getInt(); - // var length = ByteBuffer.wrap(in.readNBytes(4)).order(ByteOrder.LITTLE_ENDIAN).getInt(); - // pheader.setNowType(nowtype); - - - // if (length <= 13) { - // log.debug("[{}:{}][数据总条数:{}][当前解析第{}条][length<13][探针发包有问题]"); - // throw new Exception("数据解析异常"); - // } - - // // 读取具体数据体信息 - // byte[] bodydata = new byte[length - 13]; - // readTillLength(bodydata, length - 13); - - // // 解析数据 - // dataBean = instance.Parse(pheader, bodydata, 0); - // } else if (pheader.getTableID() == 28 || pheader.getTableID() == 29) { //28或29为Apm流统计 - - // if (pheader.getTableID() == 28) { - // length = ApmBaseDataFlow.SIZE; - // } else { - // length = BasicTrafficFlow.SIZE; - // } - - // byte[] bodydata = new byte[length]; - // readTillLength(bodydata, length); - // dataBean = instance.Parse(pheader, bodydata, 0); - - // } else if (pheader.getTableID() == 17 || pheader.getTableID() == 18) { //18 为网络性能流 - // if (pheader.getTableID() == 17) { - // length = AppFlow.SIZE; - // } else if (pheader.getTableID() == 18) { - // length = QoeFlow.SIZE; - // } - // byte[] bodydata = new byte[length]; - // readTillLength(bodydata, length); - // dataBean = instance.Parse(pheader, bodydata, 0); - // } else if (pheader.getTableID() == 20) { - // int offset = i * SstFlow.SIZE; - // dataBean = instance.Parse(pheader, unzipbodydata, offset); - // } else { - // logger.info("不需要的数据类型：" + pheader.getTableID()); - // break; - // } - // if (dataBean != null) { - // tempBaseDatas.add(dataBean); - // } - // } - - } -} From f7dbbb35cbc85a2446882dbc348a2d3b3eff8eb2 Mon Sep 17 00:00:00 2001 From: huangsimin Date: Mon, 25 Jul 2022 17:04:45 +0800 Subject: [PATCH 21/35] =?UTF-8?q?TODO:=20=E8=A7=A3=E5=86=B3=20=E4=B8=BB?= =?UTF-8?q?=E4=BB=8E=E6=93=8D=E4=BD=9C=20=E6=9B=B4=E6=96=B0=E6=B7=B7?= =?UTF-8?q?=E4=B9=B1=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 3 +- .../dataflow/controller/PacketsProcessor.java | 69 ++++++- .../dataflow/statemachine/StateMachine.java | 112 ++++------- .../statemachine/StateServerFactory.java | 185 ++---------------- .../dataflow/statemachine/SyncClosure.java | 8 +- .../{ResponseSM.java => RaftResponse.java} | 2 +- ...sor.java => SyncWorkerStateProcessor.java} | 8 +- .../rpc/annotations/ProcessorRaft.java | 1 - .../statemachine/state/StateFactory.java | 107 +++++----- .../statemachine/StateMachineTest.java | 6 +- 10 files changed, 191 insertions(+), 310 deletions(-) rename src/main/java/com/yuandian/dataflow/statemachine/rpc/{ResponseSM.java => RaftResponse.java} (93%) rename src/main/java/com/yuandian/dataflow/statemachine/rpc/{SyncConditionProcessor.java => SyncWorkerStateProcessor.java} (85%) diff --git a/pom.xml b/pom.xml index 6aa74f6..d35d4da 100644 --- a/pom.xml +++ b/pom.xml @@ -147,13 +147,14 @@ ${grpc.version} - + org.projectlombok lombok 1.18.24 provided + diff --git a/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java b/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java index ff00dc9..df20041 100644 --- a/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java +++ b/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java @@ -7,15 +7,23 @@ package com.yuandian.dataflow.controller; import java.io.Serializable; +import java.time.Instant; import java.util.ArrayList; +import com.alipay.sofa.jraft.Status; import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.error.RemotingException; +import com.alipay.sofa.jraft.rpc.InvokeCallback; import com.alipay.sofa.jraft.rpc.RpcContext; import com.alipay.sofa.jraft.rpc.RpcProcessor; import com.google.protobuf.Any; import com.yuandian.dataflow.statemachine.StateServerFactory; -import com.yuandian.dataflow.statemachine.rpc.ResponseSM; +import com.yuandian.dataflow.statemachine.SyncClosure; +import com.yuandian.dataflow.statemachine.rpc.RaftResponse; +import com.yuandian.dataflow.statemachine.rpc.SyncStateProcessor; +import com.yuandian.dataflow.statemachine.rpc.SyncWorkerStateProcessor; import com.yuandian.dataflow.statemachine.rpc.annotations.ProcessorRaft; +import com.yuandian.dataflow.statemachine.state.State; import lombok.Getter; import lombok.Setter; @@ -42,19 +50,62 @@ public class PacketsProcessor implements RpcProcessor{ + + var work = state.getWorkers().get( ss.getCluster().getServerId()); + + + work.setTaskQueueSize( work.getTaskQueueSize() - request.packets.size()); + work.setUpdateAt(Instant.now()); + + if(!ss.isLeader()) { + var requestUpdateState = new SyncWorkerStateProcessor.RequestWorkerState(); + requestUpdateState.setWorkerState(work); + log.info("转发 {}", work); + try { + ss.getRpcClient().invokeAsync(StateServerFactory.getNode().getLeaderId().getEndpoint(), + requestUpdateState, new InvokeCallback() { + @Override + public void complete(Object result, Throwable err) { + if (err != null) { + err.printStackTrace(); + } else { + log.debug("转发 udate workerState {}", result); + } + rpcCtx.sendResponse(resp); + }}, 5000); + } catch (InterruptedException | RemotingException e) { + e.printStackTrace(); + } + return; + } + + + StateServerFactory.getStateServer().applyWorkerState(work, new SyncClosure() { + @Override + public void run(Status status) { + log.debug("finsh tasks size {}, size: {}", status, request.packets.size()); + rpcCtx.sendResponse(resp); + } + + }); + }); + + } @Override public String interest() { - // TODO Auto-generated method stub return PacketsRequest.class.getName(); } - - - } diff --git a/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java b/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java index 59afa42..5cba656 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java @@ -16,8 +16,8 @@ import com.alipay.sofa.jraft.error.RaftException; import com.alipay.sofa.jraft.error.RemotingException; import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader; import com.alipay.sofa.jraft.storage.snapshot.SnapshotWriter; -import com.yuandian.dataflow.statemachine.rpc.ResponseSM; -import com.yuandian.dataflow.statemachine.rpc.SyncConditionProcessor.RequestCondition; +import com.yuandian.dataflow.statemachine.rpc.RaftResponse; +import com.yuandian.dataflow.statemachine.rpc.SyncWorkerStateProcessor.RequestWorkerState; import com.yuandian.dataflow.statemachine.rpc.SyncStateProcessor.RequestState; import com.yuandian.dataflow.statemachine.state.State; import com.yuandian.dataflow.statemachine.state.StateFactory; @@ -58,46 +58,9 @@ public class StateMachine extends StateMachineAdapter { return state; } - - /** - * Returns current value. 读取修改都在这个函数域内进行 - */ - public void useState(Function dofunc) { - dofunc.apply(this.state); - } - /** - * Returns current value. 读取修改都在这个函数域内进行 - * @throws RemotingException - * @throws InterruptedException - */ - public void updateState(Function dofunc) throws InterruptedException, RemotingException { - - var newstate = dofunc.apply(this.state); - var ss = StateServerFactory.getStateServer(); - if(!isLeader()) { - var request = new RequestState(); - request.setState(newstate); - var result = ss.getRpcClient().invokeSync(ss.getNode().getLeaderId().getEndpoint(), request, 5000); - log.info("{}", result); - return; - } - - // this.state = newstate; - if(newstate != null) { - var colsure = new SyncClosure() { - @Override - public void run(Status status) { - - } - }; - colsure.setValue(newstate); - StateServerFactory.getStateServer().applyState(newstate, colsure); - } - - } - - + + @Override @SuppressWarnings("unchecked") public void onApply(final Iterator iter) { @@ -107,7 +70,7 @@ public class StateMachine extends StateMachineAdapter { // This task is applied by this node, get value from closure to avoid additional // parsing. var closure = (SyncClosure)iter.done(); // 只支持单一个State. 全状态机只支持一种提交 - log.info("SyncDataClosure(done) taskState:{} leaderTerm:{} {}",this.state, this.leaderTerm, closure); + // log.info("SyncDataClosure(done) taskState:{} leaderTerm:{} {}",this.state, this.leaderTerm, closure); this.state = closure.getValue(); closure.success(state); closure.run(Status.OK()); @@ -115,11 +78,11 @@ public class StateMachine extends StateMachineAdapter { // Have to parse FetchAddRequest from this user log. final ByteBuffer data = iter.getData(); try { - synchronized(state) { - state = SerializerManager.getSerializer(SerializerManager.Hessian2).deserialize( - data.array(), State.class.getName()); - log.info("SyncDataClosure(null) taskState:{} leaderTerm:{}", state, this.leaderTerm); - } + + state = SerializerManager.getSerializer(SerializerManager.Hessian2).deserialize( + data.array(), State.class.getName()); + // log.info("SyncDataClosure(null) taskState:{} leaderTerm:{}", state, this.leaderTerm); + } catch (CodecException e) { e.printStackTrace(); @@ -150,20 +113,31 @@ public class StateMachine extends StateMachineAdapter { public void onLeaderStart(final long term) { log.debug("onLeaderStart {}", StateServerFactory.getCluster().getServerId()); this.leaderTerm.set(term); - try { - updateState((state)->{ - var ws = new WorkerState(StateServerFactory.getStateServer().getCluster().getServerId()); - state.getWorkers().put(ws.peerId, ws); - return state; - }); - if(!StateFactory.getMasterExecute().isAlive()) { - StateFactory.getMasterExecute().start(); - } - } catch (InterruptedException e) { - e.printStackTrace(); - } catch (RemotingException e) { - e.printStackTrace(); + + if(StateFactory.getMasterExecute().isAlive()) { + StateFactory.getMasterExecute().interrupt(); } + + var ss = StateServerFactory.getStateServer(); + ss.useFsmStateNotLock((state)->{ + var ws = state.getWorkers().get( ss.getCluster().getServerId() ); + if(ws == null) { + ws = new WorkerState(ss.getCluster().getServerId()); + state.getWorkers().put(ss.getCluster().getServerId(), ws); + } + + ss.applyState(state, new SyncClosure() { + @Override + public void run(Status status) { + log.debug("master update workerstate: {}", status); + } + + }); + + }); + + StateFactory.getMasterExecute().start(); + super.onLeaderStart(term); } @@ -177,17 +151,7 @@ public class StateMachine extends StateMachineAdapter { StateFactory.getMasterExecute().interrupt(); } - try { - updateState((state)->{ - state.getWorkers().remove( StateServerFactory.getServerId() ); - return state; - }); - - } catch (InterruptedException e) { - e.printStackTrace(); - } catch (RemotingException e) { - e.printStackTrace(); - } + } @@ -210,11 +174,11 @@ public class StateMachine extends StateMachineAdapter { var ss = StateServerFactory.getStateServer(); var ws = new WorkerState(ss.getCluster().getServerId()); - var request = new RequestCondition(); + var request = new RequestWorkerState(); request.setWorkerState(ws); log.debug("my: {} leader id {}",ss.getCluster().getServerId(), ss.getNode().getLeaderId()); - ResponseSM resp; - resp = (ResponseSM)StateServerFactory.getStateServer().getRpcClient().invokeSync(ss.getNode().getLeaderId().getEndpoint(), request, 5000); + RaftResponse resp; + resp = (RaftResponse)StateServerFactory.getStateServer().getRpcClient().invokeSync(ss.getNode().getLeaderId().getEndpoint(), request, 5000); if(resp == null) { log.debug("{} set WorkerState is error", resp); } diff --git a/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java b/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java index 4f47e9e..cef50ce 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java @@ -12,6 +12,7 @@ import java.util.Set; import java.util.concurrent.Executor; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadPoolExecutor; +import java.util.function.Consumer; import java.util.function.Function; import org.reflections.Reflections; @@ -38,8 +39,8 @@ import com.alipay.sofa.jraft.rpc.impl.BoltRaftRpcFactory; import com.alipay.sofa.jraft.util.BytesUtil; import com.alipay.sofa.jraft.util.Endpoint; import com.alipay.sofa.jraft.util.ThreadPoolUtil; -import com.yuandian.dataflow.statemachine.rpc.ResponseSM; -import com.yuandian.dataflow.statemachine.rpc.SyncConditionProcessor.RequestCondition; +import com.yuandian.dataflow.statemachine.rpc.RaftResponse; +import com.yuandian.dataflow.statemachine.rpc.SyncWorkerStateProcessor.RequestWorkerState; import com.yuandian.dataflow.statemachine.rpc.SyncStateProcessor.RequestState; import com.yuandian.dataflow.statemachine.rpc.annotations.ProcessorRaft; import com.yuandian.dataflow.statemachine.state.State; @@ -166,164 +167,21 @@ public class StateServerFactory { return this.fsm.isLeader(); } - - public void useFsmStateAsync(Function dofunc) { - - SyncClosure closure = new SyncClosure() { - @Override - public void run(Status status) { - if(status.isOk()) { - dofunc.apply(this.getValue()); - } - } - }; - - getNode().readIndex(BytesUtil.EMPTY_BYTES, new ReadIndexClosure() { - @Override - public void run(Status status, long index, byte[] reqCtx) { - - getFsm().useState((fsmState)->{ - if(status.isOk()){ - closure.setValue(fsmState); - closure.success(fsmState); - closure.run(Status.OK()); - return null; - } - - readIndexExecutor.execute(() -> { - if(isLeader()){ - log.info("Fail to get value with 'ReadIndex': {}, try to applying to the state machine.", getNode()); - applyState(fsmState, closure); - }else { - handlerNotLeaderError(closure); - } - }); - return null; - }); - } - }); - - return ; - } - - /** - * 同步更新 WorkerState - * @param dofunc - * @throws InterruptedException - * @throws RemotingException - */ - public void updateFsmWorkerState(WorkerState ws) throws InterruptedException, RemotingException { - - // leader就直接提交 - if(isLeader()) { - var closure = new SyncClosure() { - @Override - public void run(Status status) { - // log.debug("leader {}", status); - this.synclock.notify(); - } - }; - StateServerFactory.getStateServer().applyWorkerState(ws, closure); - closure.synclock.wait(5000); - return; - } - - try { - // 非leader就 rpc请求 - var ss = StateServerFactory.getStateServer(); - var request = new RequestCondition(); - request.setWorkerState(ws); - log.debug("my: {} leader id {}",ss.getCluster().getServerId(), ss.getNode().getLeaderId()); - ResponseSM resp; - resp = (ResponseSM)StateServerFactory.getRpcClient().invokeSync(StateServerFactory.getNode().getLeaderId().getEndpoint(), request, 5000); - if(resp == null) { - log.error("{} set WorkerState is error", resp); - } - // log.debug("follow is {}", resp); - return; - } catch (InterruptedException | RemotingException e) { - e.printStackTrace(); - } - } - - /** - * 同步更新整个State - * @param s - * @throws InterruptedException - * @throws RemotingException - */ - public void updateFsmState(State s) throws InterruptedException, RemotingException { - if(isLeader()) { - var closure = new SyncClosure() { - @Override - public void run(Status status) { - // log.debug("leader update {}", status); - synclock.notify(); - } - }; - StateServerFactory.getStateServer().applyState(s, closure); - closure.synclock.wait(5000); - return; - } - - var ss = StateServerFactory.getStateServer(); - var request = new RequestState(); - request.setState(s); - log.debug("my: {} leader id {}",ss.getCluster().getServerId(), ss.getNode().getLeaderId()); - ResponseSM resp; - resp = (ResponseSM)StateServerFactory.getStateServer().getRpcClient().invokeSync(ss.getNode().getLeaderId().getEndpoint(), request, 5000); - if(resp == null) { - log.error("{} set State is error", resp); - } - // log.debug("follow is {}", resp); - return; - - } - - - public void updateFsmStateAsync(State s, Function onCompleted) throws InterruptedException, RemotingException { - if(isLeader()) { - var closure = new SyncClosure() { - @Override - public void run(Status status) { - // log.debug("leader update {}", status); - if(status.isOk()) { - onCompleted.apply(status); - } - } - }; - StateServerFactory.getStateServer().applyState(s, closure); - return; - } - - var ss = StateServerFactory.getStateServer(); - var request = new RequestState(); - request.setState(s); - log.debug("my: {} leader id {}",ss.getCluster().getServerId(), ss.getNode().getLeaderId()); - StateServerFactory - .getStateServer() - .getRpcClient() - .invokeAsync(ss.getNode().getLeaderId().getEndpoint(), request, new InvokeCallback() { - @Override - public void complete(Object result, Throwable err) { - if(result != null){ - onCompleted.apply(Status.OK()); - } else { - var status = new Status(10000, "rpc invokeAsync with request: {}", request); - log.debug("{}", status); - onCompleted.apply(status); - } - } - - }, 5000); - - // log.debug("follow is {}", resp); - return; - + public void useFsmState(Consumer dofunc) { + var state = ss.fsm.getState(); + synchronized(state) { + dofunc.accept(state); + } } + public void useFsmStateNotLock(Consumer dofunc) { + var state = ss.fsm.getState(); + dofunc.accept(state); + } + + public void applyState(State state, SyncClosure closure) { // 所有的提交都必须再leader进行 @@ -351,10 +209,8 @@ public class StateServerFactory { ss.handlerNotLeaderError(closure); return; } - - - useFsmStateAsync((fsmState)->{ - + + StateServerFactory.getStateServer().useFsmStateNotLock((fsmState)->{ var wmap = fsmState.getWorkers(); var wstate = wmap.get(state.getPeerId()); @@ -376,16 +232,13 @@ public class StateServerFactory { closure.success(fsmState); closure.run(Status.OK()); } - return null; }); - - - + } - public ResponseSM redirect() { - final ResponseSM response = new ResponseSM(); + public RaftResponse redirect() { + final RaftResponse response = new RaftResponse(); response.setSuccess(false); if (this.node != null) { final PeerId leader = this.node.getLeaderId(); diff --git a/src/main/java/com/yuandian/dataflow/statemachine/SyncClosure.java b/src/main/java/com/yuandian/dataflow/statemachine/SyncClosure.java index 40ba6d9..19ab8a7 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/SyncClosure.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/SyncClosure.java @@ -3,7 +3,7 @@ package com.yuandian.dataflow.statemachine; import com.alipay.sofa.jraft.Closure; import com.alipay.sofa.jraft.Status; import com.alipay.sofa.jraft.entity.PeerId; -import com.yuandian.dataflow.statemachine.rpc.ResponseSM; +import com.yuandian.dataflow.statemachine.rpc.RaftResponse; import com.yuandian.dataflow.statemachine.state.State; import lombok.Getter; @@ -21,7 +21,7 @@ import org.slf4j.LoggerFactory; public abstract class SyncClosure implements Closure { // 状态机的统一响应 - private ResponseSM response; + private RaftResponse response; // 代表任务状态 private T value; @@ -33,7 +33,7 @@ public abstract class SyncClosure implements Closure { } public void failure(final String errorMsg, final PeerId redirect) { - final ResponseSM response = new ResponseSM(); + final RaftResponse response = new RaftResponse(); response.setSuccess(false); response.setMsg(errorMsg); response.setRedirect(redirect); @@ -41,7 +41,7 @@ public abstract class SyncClosure implements Closure { } public void success(final State value) { - final ResponseSM response = new ResponseSM(); + final RaftResponse response = new RaftResponse(); response.setState(value); response.setSuccess(true); setResponse(response); diff --git a/src/main/java/com/yuandian/dataflow/statemachine/rpc/ResponseSM.java b/src/main/java/com/yuandian/dataflow/statemachine/rpc/RaftResponse.java similarity index 93% rename from src/main/java/com/yuandian/dataflow/statemachine/rpc/ResponseSM.java rename to src/main/java/com/yuandian/dataflow/statemachine/rpc/RaftResponse.java index 5e4694f..f2769d8 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/rpc/ResponseSM.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/rpc/RaftResponse.java @@ -26,7 +26,7 @@ import lombok.extern.slf4j.Slf4j; @Getter @Setter @ToString -public class ResponseSM implements Serializable { +public class RaftResponse implements Serializable { private static final long serialVersionUID = 1L; diff --git a/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncConditionProcessor.java b/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncWorkerStateProcessor.java similarity index 85% rename from src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncConditionProcessor.java rename to src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncWorkerStateProcessor.java index 4bd0352..252f473 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncConditionProcessor.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncWorkerStateProcessor.java @@ -41,12 +41,12 @@ import lombok.extern.slf4j.Slf4j; */ @Slf4j @ProcessorRaft -public class SyncConditionProcessor implements RpcProcessor { +public class SyncWorkerStateProcessor implements RpcProcessor { @Getter @Setter @ToString - public static class RequestCondition implements Serializable { + public static class RequestWorkerState implements Serializable { private static final long serialVersionUID = 1L; @@ -54,7 +54,7 @@ public class SyncConditionProcessor implements RpcProcessor closure = new SyncClosure< State>() { @@ -70,7 +70,7 @@ public class SyncConditionProcessor implements RpcProcessor { - synchronized (alivePeers) { - alivePeers.forEach((peer) -> { - WorkerState ws = state.getWorkers().get(peer); - if (ws != null) { - var cap = 100 - ws.getTaskQueueSize(); - if (cap > 0) { - log.debug("{}", cap); - var request = new PacketsRequest(); - for (int i = 0; i < cap; i++) { - var p = Any.pack( - BacktrackingFlowOuterClass.BacktrackingFlow.newBuilder() - .setTableId(10086) - .build()); + // var state = ss.getFsm().getState(); + ss.useFsmStateNotLock((state) -> { + alivePeers.forEach((peer) -> { + WorkerState ws = state.getWorkers().get(peer); - request.getPackets().add(p); - - } - - try { - var result = ss.getRpcClient().invokeSync(ws.getPeerId().getEndpoint(), - request, 5000); - log.info("{}", result); - ws.setUpdateAt( Instant.now() ); - ws.setTaskQueueSize(ws.getTaskQueueSize() - cap); - - - - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch (RemotingException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } + if (ws != null) { + var cap = 100 - ws.getTaskQueueSize(); + log.debug("cap :{} peer: {}", cap, peer); + if (cap <= 0) { + return ; } - ss.applyState(state, new SyncClosure() { - public void run(Status status) { - log.debug("{}", status); - }; - } ); - alivePeers.notifyAll(); - }); - } + var request = new PacketsRequest(); + for (int i = 0; i < cap; i++) { + var p = Any.pack( + BacktrackingFlowOuterClass.BacktrackingFlow.newBuilder() + .setTableId(10086) + .build()); + request.getPackets().add(p); + } - return null; + + try { + log.debug("rpc {}", peer); + ss.getRpcClient().invokeAsync(peer.getEndpoint(), + request, new InvokeCallback() { + @Override + public void complete(Object result, Throwable err) { + if (err != null) { + err.printStackTrace(); + } else { + log.info("{} peer result", result); + } + } + + }, 5000); + } catch (InterruptedException | RemotingException e) { + e.printStackTrace(); + } + ws.setUpdateAt(Instant.now()); + ws.setTaskQueueSize(100); + } + }); + + ss.applyState(state, new SyncClosure() { + @Override + public void run(Status status) { + log.info("任务队列更新成功 {}", this.getValue().getWorkers()); + } + }); + + }); - synchronized (alivePeers) { - alivePeers.wait(5000); - } + + + + // ss.applyState(state, new SyncClosure() { + // public void run(Status status) { + // log.debug("{}", status); + // }; + // }); } diff --git a/src/test/java/com/yuandian/dataflow/statemachine/StateMachineTest.java b/src/test/java/com/yuandian/dataflow/statemachine/StateMachineTest.java index 565a0f2..a5e0146 100644 --- a/src/test/java/com/yuandian/dataflow/statemachine/StateMachineTest.java +++ b/src/test/java/com/yuandian/dataflow/statemachine/StateMachineTest.java @@ -9,7 +9,7 @@ import com.alipay.sofa.jraft.option.CliOptions; import com.alipay.sofa.jraft.rpc.InvokeCallback; import com.alipay.sofa.jraft.rpc.impl.BoltRaftRpcFactory; import com.alipay.sofa.jraft.util.Endpoint; -import com.yuandian.dataflow.statemachine.rpc.ResponseSM; +import com.yuandian.dataflow.statemachine.rpc.RaftResponse; import com.yuandian.dataflow.statemachine.rpc.SyncStateProcessor.RequestState; import com.yuandian.dataflow.statemachine.state.State; @@ -28,12 +28,12 @@ public class StateMachineTest { fdata.setState(fstate); var leader = new Endpoint("localhost",4441); - ResponseSM resp = (ResponseSM)rpcClient.invokeSync(leader, fdata + RaftResponse resp = (RaftResponse)rpcClient.invokeSync(leader, fdata , 5000); log.info("{}", resp); if( resp != null && !resp.isSuccess() ) { leader = resp.getRedirect().getEndpoint(); - resp = (ResponseSM)rpcClient.invokeSync(resp.getRedirect().getEndpoint(), fdata + resp = (RaftResponse)rpcClient.invokeSync(resp.getRedirect().getEndpoint(), fdata , 5000); log.info("{}", resp); } From 644e65603fe9a6a1a393911e36904ffa7a0fe4ed Mon Sep 17 00:00:00 2001 From: huangsimin Date: Mon, 25 Jul 2022 17:55:23 +0800 Subject: [PATCH 22/35] =?UTF-8?q?TODO:=20=E9=87=8D=E6=96=B0=E8=AE=BE?= =?UTF-8?q?=E8=AE=A1=20=E5=9D=87=E8=A1=A1=E5=A4=84=E7=90=86=E6=96=B9?= =?UTF-8?q?=E6=A1=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dataflow/controller/PacketsProcessor.java | 1 - .../statemachine/StateServerFactory.java | 30 +++--- .../rpc/SyncWorkerStateProcessor.java | 2 +- .../statemachine/state/StateFactory.java | 94 +++++++++++++------ 4 files changed, 79 insertions(+), 48 deletions(-) diff --git a/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java b/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java index df20041..450d47f 100644 --- a/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java +++ b/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java @@ -97,7 +97,6 @@ public class PacketsProcessor implements RpcProcessor closure) { + log.debug("applyWorkerState"); if (!ss.isLeader()) { ss.handlerNotLeaderError(closure); return; @@ -213,24 +214,21 @@ public class StateServerFactory { StateServerFactory.getStateServer().useFsmStateNotLock((fsmState)->{ var wmap = fsmState.getWorkers(); var wstate = wmap.get(state.getPeerId()); - if(wstate == null) { wmap.put(state.getPeerId(), state); - try { - final Task task = new Task(); - closure.setValue(fsmState); - task.setData(ByteBuffer.wrap(SerializerManager.getSerializer(SerializerManager.Hessian2).serialize(fsmState))); - task.setDone(closure); - StateServerFactory.getStateServer().getNode().apply(task); // 提交数据 - } catch (CodecException e) { - String errorMsg = "Fail to encode TaskState"; - log.debug("{}:{}",errorMsg, e); - closure.failure(errorMsg, PeerId.emptyPeer()); - closure.run(new Status(RaftError.EINTERNAL, errorMsg)); - } - } else { - closure.success(fsmState); - closure.run(Status.OK()); + } + + try { + final Task task = new Task(); + closure.setValue(fsmState); + task.setData(ByteBuffer.wrap(SerializerManager.getSerializer(SerializerManager.Hessian2).serialize(fsmState))); + task.setDone(closure); + StateServerFactory.getStateServer().getNode().apply(task); // 提交数据 + } catch (CodecException e) { + String errorMsg = "Fail to encode TaskState"; + log.debug("{}:{}",errorMsg, e); + closure.failure(errorMsg, PeerId.emptyPeer()); + closure.run(new Status(RaftError.EINTERNAL, errorMsg)); } }); diff --git a/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncWorkerStateProcessor.java b/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncWorkerStateProcessor.java index 252f473..a7212c6 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncWorkerStateProcessor.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncWorkerStateProcessor.java @@ -56,7 +56,7 @@ public class SyncWorkerStateProcessor implements RpcProcessor closure = new SyncClosure< State>() { @Override public void run(Status status) { diff --git a/src/main/java/com/yuandian/dataflow/statemachine/state/StateFactory.java b/src/main/java/com/yuandian/dataflow/statemachine/state/StateFactory.java index 30b8808..8c8ab68 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/state/StateFactory.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/state/StateFactory.java @@ -7,11 +7,13 @@ package com.yuandian.dataflow.statemachine.state; import java.time.Instant; +import java.util.ArrayList; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import com.alibaba.nacos.shaded.io.grpc.netty.shaded.io.netty.handler.codec.http.HttpContentEncoder.Result; import com.alipay.sofa.jraft.Status; +import com.alipay.sofa.jraft.entity.PeerId; import com.alipay.sofa.jraft.error.RemotingException; import com.alipay.sofa.jraft.rpc.InvokeCallback; import com.google.protobuf.Any; @@ -38,6 +40,18 @@ import lombok.extern.slf4j.Slf4j; @ToString public class StateFactory { + @Getter + @Setter + @ToString + public static class PeerIdCap { + private PeerId peer; + private long cap; + public PeerIdCap(PeerId pid, long cap) { + this.peer = pid; + this.cap = cap; + } + } + public static Thread masterExecute = new Thread(new Runnable() { @Override public void run() { @@ -50,46 +64,24 @@ public class StateFactory { var ss = StateServerFactory.getStateServer(); // var state = ss.getFsm().getState(); + + ArrayList pclist = new ArrayList(); + ss.useFsmStateNotLock((state) -> { alivePeers.forEach((peer) -> { WorkerState ws = state.getWorkers().get(peer); - if (ws != null) { var cap = 100 - ws.getTaskQueueSize(); log.debug("cap :{} peer: {}", cap, peer); if (cap <= 0) { return ; } - - var request = new PacketsRequest(); - for (int i = 0; i < cap; i++) { - var p = Any.pack( - BacktrackingFlowOuterClass.BacktrackingFlow.newBuilder() - .setTableId(10086) - .build()); - request.getPackets().add(p); - } - - - try { - log.debug("rpc {}", peer); - ss.getRpcClient().invokeAsync(peer.getEndpoint(), - request, new InvokeCallback() { - @Override - public void complete(Object result, Throwable err) { - if (err != null) { - err.printStackTrace(); - } else { - log.info("{} peer result", result); - } - } - - }, 5000); - } catch (InterruptedException | RemotingException e) { - e.printStackTrace(); - } ws.setUpdateAt(Instant.now()); ws.setTaskQueueSize(100); + var pc = new PeerIdCap(peer, cap); + pc.setCap(cap); + ; + pclist.add(pc); } }); @@ -97,6 +89,48 @@ public class StateFactory { @Override public void run(Status status) { log.info("任务队列更新成功 {}", this.getValue().getWorkers()); + + pclist.forEach((peer) -> { + + + + + + + if (peer.getCap() <= 0) { + return ; + } + + var request = new PacketsRequest(); + for (int i = 0; i < peer.getCap(); i++) { + var p = Any.pack( + BacktrackingFlowOuterClass.BacktrackingFlow.newBuilder() + .setTableId(10086) + .build()); + request.getPackets().add(p); + } + + + try { + log.debug("rpc {}", peer); + ss.getRpcClient().invokeAsync(peer.peer.getEndpoint() , + request, new InvokeCallback() { + @Override + public void complete(Object result, Throwable err) { + if (err != null) { + err.printStackTrace(); + } else { + log.info("{} peer result", result); + } + } + + }, 5000); + } catch (InterruptedException | RemotingException e) { + e.printStackTrace(); + } + + + }); } }); @@ -113,7 +147,7 @@ public class StateFactory { } - Thread.sleep(2000); + Thread.sleep(5000); } } catch (InterruptedException e) { e.printStackTrace(); From bf415d45e819d83994909a8f040337f9ba136ad6 Mon Sep 17 00:00:00 2001 From: huangsimin Date: Tue, 26 Jul 2022 00:25:56 +0800 Subject: [PATCH 23/35] =?UTF-8?q?TODO:=20=E5=88=86=E7=89=87=E6=A8=A1?= =?UTF-8?q?=E5=BC=8F=E5=A4=84=E7=90=86=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dataflow/controller/PacketsProcessor.java | 16 ++++++---------- .../statemachine/StateServerFactory.java | 16 +++++++++++++++- .../statemachine/state/StateFactory.java | 17 ++++++----------- 3 files changed, 27 insertions(+), 22 deletions(-) diff --git a/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java b/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java index 450d47f..84d6536 100644 --- a/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java +++ b/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java @@ -54,20 +54,17 @@ public class PacketsProcessor implements RpcProcessor{ var work = state.getWorkers().get( ss.getCluster().getServerId()); - - + + work.setTaskQueueSize( work.getTaskQueueSize() - request.packets.size()); work.setUpdateAt(Instant.now()); - + log.debug("workerState taskQueueSize: {} psize: {}", work.getTaskQueueSize(), request.packets.size()); if(!ss.isLeader()) { var requestUpdateState = new SyncWorkerStateProcessor.RequestWorkerState(); requestUpdateState.setWorkerState(work); @@ -80,7 +77,7 @@ public class PacketsProcessor implements RpcProcessor() { @Override public void run(Status status) { diff --git a/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java b/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java index 6f888d1..be327a4 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java @@ -177,8 +177,22 @@ public class StateServerFactory { } public void useFsmStateNotLock(Consumer dofunc) { + var state = ss.fsm.getState(); - dofunc.accept(state); + synchronized(state) { + dofunc.accept(state); + } + + // getNode().readIndex(BytesUtil.EMPTY_BYTES, new ReadIndexClosure() { + + // @Override + // public void run(Status status, long index, byte[] reqCtx) { + // var state = ss.fsm.getState(); + // dofunc.accept(state); + // } + + // } ); + } diff --git a/src/main/java/com/yuandian/dataflow/statemachine/state/StateFactory.java b/src/main/java/com/yuandian/dataflow/statemachine/state/StateFactory.java index 8c8ab68..c95931f 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/state/StateFactory.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/state/StateFactory.java @@ -90,19 +90,14 @@ public class StateFactory { public void run(Status status) { log.info("任务队列更新成功 {}", this.getValue().getWorkers()); - pclist.forEach((peer) -> { - - - - - - - if (peer.getCap() <= 0) { + pclist.forEach((peercap) -> { + + if (peercap.getCap() <= 0) { return ; } var request = new PacketsRequest(); - for (int i = 0; i < peer.getCap(); i++) { + for (int i = 0; i < peercap.getCap(); i++) { var p = Any.pack( BacktrackingFlowOuterClass.BacktrackingFlow.newBuilder() .setTableId(10086) @@ -112,8 +107,8 @@ public class StateFactory { try { - log.debug("rpc {}", peer); - ss.getRpcClient().invokeAsync(peer.peer.getEndpoint() , + log.debug("rpc {}", peercap); + ss.getRpcClient().invokeAsync(peercap.peer.getEndpoint() , request, new InvokeCallback() { @Override public void complete(Object result, Throwable err) { From 849821fd8b65802b289b4b4b3af1bde221ecb393 Mon Sep 17 00:00:00 2001 From: huangsimin <474420502@qq.com> Date: Tue, 26 Jul 2022 16:59:00 +0800 Subject: [PATCH 24/35] =?UTF-8?q?TODO:=20=E8=A7=A3=E5=86=B3=E5=A4=84?= =?UTF-8?q?=E7=90=86packets=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dataflow/controller/PacketsProcessor.java | 39 +++---- .../dataflow/grpc/CollectPackets.java | 26 ++--- .../{SyncClosure.java => OperateClosure.java} | 13 +-- .../dataflow/statemachine/StateMachine.java | 102 +++++++++++++----- .../statemachine/StateServerFactory.java | 71 ++++-------- .../statemachine/operate/Operate.java | 82 ++++++++++++++ ...teProcessor.java => OperateProcessor.java} | 17 +-- .../statemachine/rpc/RaftResponse.java | 3 +- .../rpc/SyncWorkerStateProcessor.java | 78 -------------- .../statemachine/state/StateFactory.java | 93 +++++++--------- .../com/yuandian/dataflow/MongodbTest.java | 6 +- .../statemachine/StateMachineTest.java | 82 +++++++------- 12 files changed, 307 insertions(+), 305 deletions(-) rename src/main/java/com/yuandian/dataflow/statemachine/{SyncClosure.java => OperateClosure.java} (79%) create mode 100644 src/main/java/com/yuandian/dataflow/statemachine/operate/Operate.java rename src/main/java/com/yuandian/dataflow/statemachine/rpc/{SyncStateProcessor.java => OperateProcessor.java} (74%) delete mode 100644 src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncWorkerStateProcessor.java diff --git a/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java b/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java index 84d6536..a19e406 100644 --- a/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java +++ b/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java @@ -18,10 +18,12 @@ import com.alipay.sofa.jraft.rpc.RpcContext; import com.alipay.sofa.jraft.rpc.RpcProcessor; import com.google.protobuf.Any; import com.yuandian.dataflow.statemachine.StateServerFactory; -import com.yuandian.dataflow.statemachine.SyncClosure; +import com.yuandian.dataflow.statemachine.operate.Operate; +import com.yuandian.dataflow.statemachine.operate.Operate.OperateType; +import com.yuandian.dataflow.statemachine.OperateClosure; import com.yuandian.dataflow.statemachine.rpc.RaftResponse; -import com.yuandian.dataflow.statemachine.rpc.SyncStateProcessor; -import com.yuandian.dataflow.statemachine.rpc.SyncWorkerStateProcessor; +import com.yuandian.dataflow.statemachine.rpc.OperateProcessor; + import com.yuandian.dataflow.statemachine.rpc.annotations.ProcessorRaft; import com.yuandian.dataflow.statemachine.state.State; @@ -65,34 +67,17 @@ public class PacketsProcessor implements RpcProcessor() { @Override public void run(Status status) { - log.debug("finsh tasks size {}, size: {}", status, request.packets.size()); - rpcCtx.sendResponse(resp); + log.info("{}", this.getResponse()); + rpcCtx.sendResponse(this.getResponse()); } + }); }); diff --git a/src/main/java/com/yuandian/dataflow/grpc/CollectPackets.java b/src/main/java/com/yuandian/dataflow/grpc/CollectPackets.java index f751c86..0395897 100644 --- a/src/main/java/com/yuandian/dataflow/grpc/CollectPackets.java +++ b/src/main/java/com/yuandian/dataflow/grpc/CollectPackets.java @@ -81,7 +81,7 @@ public class CollectPackets extends CollectPacketsServerImplBase { log.info("次序:{} 条数: {}, {}:ms", count, i, Duration.between(now, Instant.now()).toMillis()); } } catch (Exception e) { - e.printStackTrace(); + log.info("{}", e.toString()); } finally { channel.shutdown().awaitTermination(1, TimeUnit.SECONDS); } @@ -99,7 +99,7 @@ public class CollectPackets extends CollectPacketsServerImplBase { // MongodbTest.insertMsgToMongoDB(result); } catch (InvalidProtocolBufferException e) { - e.printStackTrace(); + log.info("{}", e.toString()); } return null; }); @@ -110,7 +110,7 @@ public class CollectPackets extends CollectPacketsServerImplBase { // MongodbTest.insertMsgToMongoDB(result); } catch (InvalidProtocolBufferException e) { - e.printStackTrace(); + log.info("{}", e.toString()); } return null; });*/ @@ -131,7 +131,7 @@ public class CollectPackets extends CollectPacketsServerImplBase { // System.out.println("result:" + response.getBody()); } catch (InvalidProtocolBufferException e) { - e.printStackTrace(); + log.info("{}", e.toString()); } return null; }); @@ -143,7 +143,7 @@ public class CollectPackets extends CollectPacketsServerImplBase { // MongodbTest.insertMsgToMongoDB(result); } catch (InvalidProtocolBufferException e) { - e.printStackTrace(); + log.info("{}", e.toString()); } return null; }); @@ -154,7 +154,7 @@ public class CollectPackets extends CollectPacketsServerImplBase { // MongodbTest.insertMsgToMongoDB(result); } catch (InvalidProtocolBufferException e) { - e.printStackTrace(); + log.info("{}", e.toString()); } return null; }); @@ -165,7 +165,7 @@ public class CollectPackets extends CollectPacketsServerImplBase { // MongodbTest.insertMsgToMongoDB(result); } catch (InvalidProtocolBufferException e) { - e.printStackTrace(); + log.info("{}", e.toString()); } return null; }); @@ -176,7 +176,7 @@ public class CollectPackets extends CollectPacketsServerImplBase { // MongodbTest.insertMsgToMongoDB(result); } catch (InvalidProtocolBufferException e) { - e.printStackTrace(); + log.info("{}", e.toString()); } return null; }); @@ -187,7 +187,7 @@ public class CollectPackets extends CollectPacketsServerImplBase { // MongodbTest.insertMsgToMongoDB(result); } catch (InvalidProtocolBufferException e) { - e.printStackTrace(); + log.info("{}", e.toString()); } return null; }); @@ -198,7 +198,7 @@ public class CollectPackets extends CollectPacketsServerImplBase { // MongodbTest.insertMsgToMongoDB(result); } catch (InvalidProtocolBufferException e) { - e.printStackTrace(); + log.info("{}", e.toString()); } return null; }); @@ -209,7 +209,7 @@ public class CollectPackets extends CollectPacketsServerImplBase { // MongodbTest.insertMsgToMongoDB(result); } catch (InvalidProtocolBufferException e) { - e.printStackTrace(); + log.info("{}", e.toString()); } return null; }); @@ -220,7 +220,7 @@ public class CollectPackets extends CollectPacketsServerImplBase { // MongodbTest.insertMsgToMongoDB(result); } catch (InvalidProtocolBufferException e) { - e.printStackTrace(); + log.info("{}", e.toString()); } return null; }); @@ -231,7 +231,7 @@ public class CollectPackets extends CollectPacketsServerImplBase { // MongodbTest.insertMsgToMongoDB(result); } catch (InvalidProtocolBufferException e) { - e.printStackTrace(); + log.info("{}", e.toString()); } return null; });*/ diff --git a/src/main/java/com/yuandian/dataflow/statemachine/SyncClosure.java b/src/main/java/com/yuandian/dataflow/statemachine/OperateClosure.java similarity index 79% rename from src/main/java/com/yuandian/dataflow/statemachine/SyncClosure.java rename to src/main/java/com/yuandian/dataflow/statemachine/OperateClosure.java index 19ab8a7..69a10e7 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/SyncClosure.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/OperateClosure.java @@ -3,6 +3,7 @@ package com.yuandian.dataflow.statemachine; import com.alipay.sofa.jraft.Closure; import com.alipay.sofa.jraft.Status; import com.alipay.sofa.jraft.entity.PeerId; +import com.yuandian.dataflow.statemachine.operate.Operate; import com.yuandian.dataflow.statemachine.rpc.RaftResponse; import com.yuandian.dataflow.statemachine.state.State; @@ -18,17 +19,17 @@ import org.slf4j.LoggerFactory; @Getter @Setter @ToString -public abstract class SyncClosure implements Closure { +public abstract class OperateClosure implements Closure { // 状态机的统一响应 private RaftResponse response; // 代表任务状态 - private T value; + private Operate value; - public Object synclock = new Object(); + - public SyncClosure() { + public OperateClosure() { } @@ -40,9 +41,9 @@ public abstract class SyncClosure implements Closure { setResponse(response); } - public void success(final State value) { + public void success(final Operate value) { final RaftResponse response = new RaftResponse(); - response.setState(value); + response.setOperate(value); response.setSuccess(true); setResponse(response); } diff --git a/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java b/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java index 5cba656..78589c1 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java @@ -16,9 +16,11 @@ import com.alipay.sofa.jraft.error.RaftException; import com.alipay.sofa.jraft.error.RemotingException; import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader; import com.alipay.sofa.jraft.storage.snapshot.SnapshotWriter; +import com.yuandian.dataflow.statemachine.operate.Operate; +import com.yuandian.dataflow.statemachine.operate.Operate.OperateType; import com.yuandian.dataflow.statemachine.rpc.RaftResponse; -import com.yuandian.dataflow.statemachine.rpc.SyncWorkerStateProcessor.RequestWorkerState; -import com.yuandian.dataflow.statemachine.rpc.SyncStateProcessor.RequestState; + +import com.yuandian.dataflow.statemachine.rpc.OperateProcessor.RequestOperate; import com.yuandian.dataflow.statemachine.state.State; import com.yuandian.dataflow.statemachine.state.StateFactory; import com.yuandian.dataflow.statemachine.state.WorkerState; @@ -58,38 +60,64 @@ public class StateMachine extends StateMachineAdapter { return state; } - - - @Override @SuppressWarnings("unchecked") public void onApply(final Iterator iter) { while (iter.hasNext()) { + Operate op = null; + OperateClosure closure = null; if (iter.done() != null) { // This task is applied by this node, get value from closure to avoid additional // parsing. - var closure = (SyncClosure)iter.done(); // 只支持单一个State. 全状态机只支持一种提交 + closure = (OperateClosure)iter.done(); // 只支持单一个State. 全状态机只支持一种提交 // log.info("SyncDataClosure(done) taskState:{} leaderTerm:{} {}",this.state, this.leaderTerm, closure); - this.state = closure.getValue(); - closure.success(state); - closure.run(Status.OK()); + op = closure.getValue(); + } else { // Have to parse FetchAddRequest from this user log. final ByteBuffer data = iter.getData(); try { - state = SerializerManager.getSerializer(SerializerManager.Hessian2).deserialize( - data.array(), State.class.getName()); + op = SerializerManager.getSerializer(SerializerManager.Hessian2).deserialize( + data.array(),Operate.class.getName()); // log.info("SyncDataClosure(null) taskState:{} leaderTerm:{}", state, this.leaderTerm); - + } catch (CodecException e) { - e.printStackTrace(); + log.info("{}", e.toString()); } } + if(op != null) { + switch(op.getType()) { + case PUT: + WorkerState ws = op.getValue(); + state.getWorkers().put(ws.peerId, ws); + if(closure != null) { + closure.success(op); + closure.run(Status.OK()); + } + break; + case REMOVE: + + + if(closure != null) { + closure.success(op); + closure.run(Status.OK()); + } + break; + default: + break; + + } + } else { + + } + + + iter.next(); } } @@ -123,15 +151,16 @@ public class StateMachine extends StateMachineAdapter { var ws = state.getWorkers().get( ss.getCluster().getServerId() ); if(ws == null) { ws = new WorkerState(ss.getCluster().getServerId()); - state.getWorkers().put(ss.getCluster().getServerId(), ws); + // state.getWorkers().put(ss.getCluster().getServerId(), ws); } - ss.applyState(state, new SyncClosure() { + Operate op = new Operate(OperateType.PUT); + op.setValue(ws); + ss.applyOperate(op, new OperateClosure() { @Override public void run(Status status) { log.debug("master update workerstate: {}", status); } - }); }); @@ -174,18 +203,22 @@ public class StateMachine extends StateMachineAdapter { var ss = StateServerFactory.getStateServer(); var ws = new WorkerState(ss.getCluster().getServerId()); - var request = new RequestWorkerState(); - request.setWorkerState(ws); + log.debug("my: {} leader id {}",ss.getCluster().getServerId(), ss.getNode().getLeaderId()); - RaftResponse resp; - resp = (RaftResponse)StateServerFactory.getStateServer().getRpcClient().invokeSync(ss.getNode().getLeaderId().getEndpoint(), request, 5000); - if(resp == null) { - log.debug("{} set WorkerState is error", resp); - } - log.debug("WorkerState is {}", resp); + + var op = new Operate(OperateType.PUT); + op.setValue(ws); + + Operate.CallOperate(op, new OperateClosure() { + @Override + public void run(Status status) { + log.info("{} {}", status, this.getResponse()); + } + }); + return; - } catch (InterruptedException | RemotingException e) { - e.printStackTrace(); + } catch (Exception e) { + log.info("{}", e.toString()); } super.onStartFollowing(ctx); @@ -199,6 +232,23 @@ public class StateMachine extends StateMachineAdapter { @Override public void onStopFollowing(LeaderChangeContext ctx) { log.debug("{} {}",ctx, StateServerFactory.getCluster().getServerId()); + + var ss = StateServerFactory.getStateServer(); + var ws = new WorkerState(ss.getCluster().getServerId()); + + log.debug("my: {} leader id {}",ss.getCluster().getServerId(), ss.getNode().getLeaderId()); + + var op = new Operate(OperateType.PUT); + op.setValue(ws); + + Operate.CallOperate(op, new OperateClosure() { + @Override + public void run(Status status) { + log.info("{} {}", status, this.getResponse()); + } + }); + + super.onStopFollowing(ctx); } diff --git a/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java b/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java index be327a4..69ddde0 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java @@ -39,9 +39,10 @@ import com.alipay.sofa.jraft.rpc.impl.BoltRaftRpcFactory; import com.alipay.sofa.jraft.util.BytesUtil; import com.alipay.sofa.jraft.util.Endpoint; import com.alipay.sofa.jraft.util.ThreadPoolUtil; +import com.yuandian.dataflow.statemachine.operate.Operate; import com.yuandian.dataflow.statemachine.rpc.RaftResponse; -import com.yuandian.dataflow.statemachine.rpc.SyncWorkerStateProcessor.RequestWorkerState; -import com.yuandian.dataflow.statemachine.rpc.SyncStateProcessor.RequestState; + +import com.yuandian.dataflow.statemachine.rpc.OperateProcessor.RequestOperate; import com.yuandian.dataflow.statemachine.rpc.annotations.ProcessorRaft; import com.yuandian.dataflow.statemachine.state.State; import com.yuandian.dataflow.statemachine.state.WorkerState; @@ -150,9 +151,9 @@ public class StateServerFactory { try { cluster.getRpcServer().registerProcessor((RpcProcessor) pRaftClass.newInstance()); } catch (InstantiationException e) { - e.printStackTrace(); + log.info("{}", e.toString()); } catch (IllegalAccessException e) { - e.printStackTrace(); + log.info("{}", e.toString()); } }); node = cluster.start(); @@ -178,26 +179,24 @@ public class StateServerFactory { public void useFsmStateNotLock(Consumer dofunc) { - var state = ss.fsm.getState(); - synchronized(state) { - dofunc.accept(state); - } + + - // getNode().readIndex(BytesUtil.EMPTY_BYTES, new ReadIndexClosure() { + getNode().readIndex(BytesUtil.EMPTY_BYTES, new ReadIndexClosure() { - // @Override - // public void run(Status status, long index, byte[] reqCtx) { - // var state = ss.fsm.getState(); - // dofunc.accept(state); - // } + @Override + public void run(Status status, long index, byte[] reqCtx) { + var state = ss.fsm.getState(); + dofunc.accept(state); + } - // } ); + } ); } - public void applyState(State state, SyncClosure closure) { + public void applyOperate(Operate op, OperateClosure closure) { // 所有的提交都必须再leader进行 if (!ss.isLeader()) { ss.handlerNotLeaderError(closure); @@ -205,9 +204,10 @@ public class StateServerFactory { } try { - closure.setValue(state); + + closure.setValue(op); final Task task = new Task(); - task.setData(ByteBuffer.wrap(SerializerManager.getSerializer(SerializerManager.Hessian2).serialize(state))); + task.setData(ByteBuffer.wrap(SerializerManager.getSerializer(SerializerManager.Hessian2).serialize(op))); task.setDone(closure); // 确认所有数据 一致, 不需要加锁 StateServerFactory.getStateServer().getNode().apply(task); } catch (CodecException e) { @@ -218,36 +218,7 @@ public class StateServerFactory { } } - public void applyWorkerState(WorkerState state, SyncClosure closure) { - log.debug("applyWorkerState"); - if (!ss.isLeader()) { - ss.handlerNotLeaderError(closure); - return; - } - - StateServerFactory.getStateServer().useFsmStateNotLock((fsmState)->{ - var wmap = fsmState.getWorkers(); - var wstate = wmap.get(state.getPeerId()); - if(wstate == null) { - wmap.put(state.getPeerId(), state); - } - - try { - final Task task = new Task(); - closure.setValue(fsmState); - task.setData(ByteBuffer.wrap(SerializerManager.getSerializer(SerializerManager.Hessian2).serialize(fsmState))); - task.setDone(closure); - StateServerFactory.getStateServer().getNode().apply(task); // 提交数据 - } catch (CodecException e) { - String errorMsg = "Fail to encode TaskState"; - log.debug("{}:{}",errorMsg, e); - closure.failure(errorMsg, PeerId.emptyPeer()); - closure.run(new Status(RaftError.EINTERNAL, errorMsg)); - } - }); - - - } + public RaftResponse redirect() { final RaftResponse response = new RaftResponse(); @@ -261,7 +232,7 @@ public class StateServerFactory { return response; } - public void handlerNotLeaderError(final SyncClosure closure) { + public void handlerNotLeaderError(final OperateClosure closure) { closure.failure("Not leader.", redirect().getRedirect()); closure.run(new Status(RaftError.EPERM, "Not leader")); } @@ -289,7 +260,7 @@ public class StateServerFactory { public static void main(String[] args) throws InterruptedException, RemotingException { var rpcClient = new BoltRaftRpcFactory().createRpcClient(); rpcClient.init(new CliOptions()); - var resp = rpcClient.invokeSync(new Endpoint("localhost",4441), new RequestState(), 5000); + var resp = rpcClient.invokeSync(new Endpoint("localhost",4441), new RequestOperate(), 5000); log.info("{}", resp); } } diff --git a/src/main/java/com/yuandian/dataflow/statemachine/operate/Operate.java b/src/main/java/com/yuandian/dataflow/statemachine/operate/Operate.java new file mode 100644 index 0000000..d798fdf --- /dev/null +++ b/src/main/java/com/yuandian/dataflow/statemachine/operate/Operate.java @@ -0,0 +1,82 @@ +package com.yuandian.dataflow.statemachine.operate; + +import java.io.Serializable; + +import com.alipay.sofa.jraft.Status; +import com.alipay.sofa.jraft.error.RemotingException; +import com.alipay.sofa.jraft.rpc.InvokeCallback; +import com.yuandian.dataflow.projo.Response; +import com.yuandian.dataflow.statemachine.OperateClosure; +import com.yuandian.dataflow.statemachine.StateServerFactory; +import com.yuandian.dataflow.statemachine.rpc.OperateProcessor; +import com.yuandian.dataflow.statemachine.rpc.RaftResponse; +import com.yuandian.dataflow.statemachine.state.StateFactory; + + +import lombok.Data; +import lombok.var; +import lombok.extern.slf4j.Slf4j; + +/** + * 操作 + * + * @author eson + */ +@Slf4j +@Data +@var +public class Operate implements Serializable { + + public static enum OperateType { + PUT, REMOVE; + } + + private OperateType type; + private Object value; + + public Operate(OperateType t) { + this.type = t; + } + + public T getValue() { + return (T) this.value; + }; + + public void setValue(T value) { + this.value = value; + return; + }; + + public static void CallOperate(Operate op, OperateClosure closure) { + + var ss = StateServerFactory.getStateServer(); + if (StateServerFactory.getStateServer().isLeader()) { + ss.applyOperate(op, closure); + return; + } + + var request = new OperateProcessor.RequestOperate(); + request.setOperate(op); + + var leaderId = StateServerFactory.getStateServer().getNode().getLeaderId(); + try { + ss.getRpcClient().invokeAsync(leaderId.getEndpoint(), + request, new InvokeCallback() { + + @Override + public void complete(Object result, Throwable err) { + log.info("{}", result); + var resp = (RaftResponse)result; + closure.setResponse(resp); + closure.success(resp.getOperate()); + } + + }, 5000); + } catch (InterruptedException | RemotingException e) { + // TODO Auto-generated catch block + closure.failure("failure", null); + log.info("{}", e.toString()); + } + + } +} diff --git a/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncStateProcessor.java b/src/main/java/com/yuandian/dataflow/statemachine/rpc/OperateProcessor.java similarity index 74% rename from src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncStateProcessor.java rename to src/main/java/com/yuandian/dataflow/statemachine/rpc/OperateProcessor.java index 6bd48e8..6c96a28 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncStateProcessor.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/rpc/OperateProcessor.java @@ -17,8 +17,9 @@ import com.alipay.sofa.jraft.error.RaftError; import com.alipay.sofa.jraft.rpc.RpcContext; import com.alipay.sofa.jraft.rpc.RpcProcessor; import com.yuandian.dataflow.statemachine.StateServerFactory; -import com.yuandian.dataflow.statemachine.SyncClosure; +import com.yuandian.dataflow.statemachine.OperateClosure; import com.yuandian.dataflow.statemachine.StateServerFactory.StateServer; +import com.yuandian.dataflow.statemachine.operate.Operate; import com.yuandian.dataflow.statemachine.rpc.annotations.ProcessorRaft; import com.yuandian.dataflow.statemachine.state.State; import com.alipay.sofa.jraft.entity.PeerId; @@ -39,7 +40,7 @@ import lombok.extern.slf4j.Slf4j; */ @Slf4j @ProcessorRaft -public class SyncStateProcessor implements RpcProcessor { +public class OperateProcessor implements RpcProcessor { /** * 同步状态时, 需要用的结构类. 新增的状态可以在 State结构里添加 @@ -51,20 +52,20 @@ public class SyncStateProcessor implements RpcProcessor closure = new SyncClosure() { + final OperateClosure closure = new OperateClosure() { @Override public void run(Status status) { rpcCtx.sendResponse(getResponse()); @@ -72,12 +73,12 @@ public class SyncStateProcessor implements RpcProcessor { - - @Getter - @Setter - @ToString - public static class RequestWorkerState implements Serializable { - - private static final long serialVersionUID = 1L; - - private WorkerState workerState; - } - - @Override - public void handleRequest(RpcContext rpcCtx, RequestWorkerState request) { - - log.info("RequestWorkerState: {}", request); - final SyncClosure closure = new SyncClosure< State>() { - @Override - public void run(Status status) { - rpcCtx.sendResponse(getResponse()); - log.info("{}", status); - } - }; - - StateServerFactory.getStateServer().applyWorkerState(request.getWorkerState(), closure); - } - - @Override - public String interest() { - return RequestWorkerState.class.getName(); - } - - - -} diff --git a/src/main/java/com/yuandian/dataflow/statemachine/state/StateFactory.java b/src/main/java/com/yuandian/dataflow/statemachine/state/StateFactory.java index c95931f..d6068a0 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/state/StateFactory.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/state/StateFactory.java @@ -20,7 +20,9 @@ import com.google.protobuf.Any; import com.yuandian.dataflow.controller.PacketsProcessor.PacketsRequest; import com.yuandian.dataflow.proto.msgtype.BacktrackingFlowOuterClass; import com.yuandian.dataflow.statemachine.StateServerFactory; -import com.yuandian.dataflow.statemachine.SyncClosure; +import com.yuandian.dataflow.statemachine.operate.Operate; +import com.yuandian.dataflow.statemachine.operate.Operate.OperateType; +import com.yuandian.dataflow.statemachine.OperateClosure; import lombok.Getter; import lombok.Setter; @@ -43,9 +45,10 @@ public class StateFactory { @Getter @Setter @ToString - public static class PeerIdCap { + public static class PeerIdCap { private PeerId peer; private long cap; + public PeerIdCap(PeerId pid, long cap) { this.peer = pid; this.cap = cap; @@ -61,12 +64,10 @@ public class StateFactory { var alivePeers = StateServerFactory.getCluster().getRaftNode().listAlivePeers(); log.debug("master execute {}", StateServerFactory.getCluster().getRaftNode().listAlivePeers()); if (alivePeers != null) { - + var ss = StateServerFactory.getStateServer(); // var state = ss.getFsm().getState(); - ArrayList pclist = new ArrayList(); - ss.useFsmStateNotLock((state) -> { alivePeers.forEach((peer) -> { WorkerState ws = state.getWorkers().get(peer); @@ -74,70 +75,54 @@ public class StateFactory { var cap = 100 - ws.getTaskQueueSize(); log.debug("cap :{} peer: {}", cap, peer); if (cap <= 0) { - return ; + return; } ws.setUpdateAt(Instant.now()); ws.setTaskQueueSize(100); - var pc = new PeerIdCap(peer, cap); - pc.setCap(cap); - ; - pclist.add(pc); - } - }); - ss.applyState(state, new SyncClosure() { - @Override - public void run(Status status) { - log.info("任务队列更新成功 {}", this.getValue().getWorkers()); + var op = new Operate(OperateType.PUT); + op.setValue(ws); - pclist.forEach((peercap) -> { - - if (peercap.getCap() <= 0) { - return ; - } - - var request = new PacketsRequest(); - for (int i = 0; i < peercap.getCap(); i++) { - var p = Any.pack( - BacktrackingFlowOuterClass.BacktrackingFlow.newBuilder() - .setTableId(10086) - .build()); - request.getPackets().add(p); - } - - - try { - log.debug("rpc {}", peercap); - ss.getRpcClient().invokeAsync(peercap.peer.getEndpoint() , + var request = new PacketsRequest(); + for (int i = 0; i < cap; i++) { + var p = Any.pack( + BacktrackingFlowOuterClass.BacktrackingFlow.newBuilder() + .setTableId(10086) + .build()); + request.getPackets().add(p); + } + + // Operate.CallOperate(op, new OperateClosure() { + // @Override + // public void run(Status status) { + // // TODO Auto-generated method stub + // log.info("{}", status); + // } + // }); + + try { + ss.getRpcClient().invokeAsync(peer.getEndpoint(), request, new InvokeCallback() { + @Override public void complete(Object result, Throwable err) { - if (err != null) { - err.printStackTrace(); - } else { - log.info("{} peer result", result); - } + log.info("{}", result); } - + }, 5000); - } catch (InterruptedException | RemotingException e) { - e.printStackTrace(); - } - - - }); + } catch (InterruptedException | RemotingException e) { + log.info("error send packets {}", e.toString()); + } + } }); - }); - - // ss.applyState(state, new SyncClosure() { - // public void run(Status status) { - // log.debug("{}", status); - // }; + // public void run(Status status) { + // log.debug("{}", status); + // }; // }); } @@ -145,7 +130,7 @@ public class StateFactory { Thread.sleep(5000); } } catch (InterruptedException e) { - e.printStackTrace(); + log.info("{}", e.toString()); } } diff --git a/src/test/java/com/yuandian/dataflow/MongodbTest.java b/src/test/java/com/yuandian/dataflow/MongodbTest.java index 73e1ed9..7822bd7 100644 --- a/src/test/java/com/yuandian/dataflow/MongodbTest.java +++ b/src/test/java/com/yuandian/dataflow/MongodbTest.java @@ -5,12 +5,16 @@ import com.mongodb.MongoCredential; import com.mongodb.ServerAddress; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; + +import lombok.extern.slf4j.Slf4j; + import org.bson.Document; import java.lang.reflect.Field; import java.util.ArrayList; import java.util.List; +@Slf4j public class MongodbTest { public static void insertMsgToMongoDB(T obj) { @@ -34,7 +38,7 @@ public class MongodbTest { System.err.println("insert success"); } catch (Exception e) { - e.printStackTrace(); + log.info("{}", e.toString()); } } diff --git a/src/test/java/com/yuandian/dataflow/statemachine/StateMachineTest.java b/src/test/java/com/yuandian/dataflow/statemachine/StateMachineTest.java index a5e0146..eed9347 100644 --- a/src/test/java/com/yuandian/dataflow/statemachine/StateMachineTest.java +++ b/src/test/java/com/yuandian/dataflow/statemachine/StateMachineTest.java @@ -10,7 +10,7 @@ import com.alipay.sofa.jraft.rpc.InvokeCallback; import com.alipay.sofa.jraft.rpc.impl.BoltRaftRpcFactory; import com.alipay.sofa.jraft.util.Endpoint; import com.yuandian.dataflow.statemachine.rpc.RaftResponse; -import com.yuandian.dataflow.statemachine.rpc.SyncStateProcessor.RequestState; +import com.yuandian.dataflow.statemachine.rpc.OperateProcessor.RequestOperate; import com.yuandian.dataflow.statemachine.state.State; import lombok.var; @@ -20,53 +20,53 @@ import lombok.extern.slf4j.Slf4j; public class StateMachineTest { @Test void testOnApply() throws InterruptedException, RemotingException { - var rpcClient = new BoltRaftRpcFactory().createRpcClient(); - rpcClient.init(new CliOptions()); + // var rpcClient = new BoltRaftRpcFactory().createRpcClient(); + // rpcClient.init(new CliOptions()); - var fstate = new State(); - var fdata = new RequestState(); - fdata.setState(fstate); + // var fstate = new State(); + // var fdata = new RequestOperate(); + // fdata.setOperate(fstate); - var leader = new Endpoint("localhost",4441); - RaftResponse resp = (RaftResponse)rpcClient.invokeSync(leader, fdata - , 5000); - log.info("{}", resp); - if( resp != null && !resp.isSuccess() ) { - leader = resp.getRedirect().getEndpoint(); - resp = (RaftResponse)rpcClient.invokeSync(resp.getRedirect().getEndpoint(), fdata - , 5000); - log.info("{}", resp); - } + // var leader = new Endpoint("localhost",4441); + // RaftResponse resp = (RaftResponse)rpcClient.invokeSync(leader, fdata + // , 5000); + // log.info("{}", resp); + // if( resp != null && !resp.isSuccess() ) { + // leader = resp.getRedirect().getEndpoint(); + // resp = (RaftResponse)rpcClient.invokeSync(resp.getRedirect().getEndpoint(), fdata + // , 5000); + // log.info("{}", resp); + // } - int i = 0 ; - while(true) { + // int i = 0 ; + // while(true) { - var state = new State(); - var request = new RequestState(); // 创建请求 - request.setState(state); // 添加请求的参数 + // var state = new State(); + // var request = new RequestOperate(); // 创建请求 + // request.setState(state); // 添加请求的参数 - var wstate = state.getWorkers(); + // var wstate = state.getWorkers(); - // state.getWorker().setPeerId( PeerId.parsePeer("localhost:2222") ); - // state.getWorker().setTaskQueueSize(i); + // // state.getWorker().setPeerId( PeerId.parsePeer("localhost:2222") ); + // // state.getWorker().setTaskQueueSize(i); - var pi = i ; - i++; - if (i >= 1000) { - break; - } - rpcClient.invokeAsync(leader, request, new InvokeCallback() { - @Override - public void complete(Object result, Throwable err) { - // ResponseSM resp = (ResponseSM)result; - log.info("{} {} {}", result, err, pi); - } + // var pi = i ; + // i++; + // if (i >= 1000) { + // break; + // } + // rpcClient.invokeAsync(leader, request, new InvokeCallback() { + // @Override + // public void complete(Object result, Throwable err) { + // // ResponseSM resp = (ResponseSM)result; + // log.info("{} {} {}", result, err, pi); + // } - @Override - public Executor executor() { - return null; - } - } , 5000); - } + // @Override + // public Executor executor() { + // return null; + // } + // } , 5000); + // } } } From 159e25ddc67ae7408840ee424b7c969dcf54fecd Mon Sep 17 00:00:00 2001 From: huangsimin <474420502@qq.com> Date: Tue, 26 Jul 2022 17:32:51 +0800 Subject: [PATCH 25/35] =?UTF-8?q?TODO:=20=E5=AE=8C=E5=96=84=E9=80=BB?= =?UTF-8?q?=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dataflow/controller/PacketsProcessor.java | 7 +++- .../statemachine/state/StateFactory.java | 41 +++++++++---------- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java b/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java index a19e406..f2db24b 100644 --- a/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java +++ b/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java @@ -52,13 +52,14 @@ public class PacketsProcessor implements RpcProcessor{ var work = state.getWorkers().get( ss.getCluster().getServerId()); @@ -75,7 +76,9 @@ public class PacketsProcessor implements RpcProcessor Date: Tue, 26 Jul 2022 17:52:58 +0800 Subject: [PATCH 26/35] =?UTF-8?q?=E5=87=8F=E5=B0=91=E5=9B=9E=E8=B0=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/yuandian/dataflow/controller/PacketsProcessor.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java b/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java index f2db24b..e9f6d0a 100644 --- a/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java +++ b/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java @@ -67,17 +67,16 @@ public class PacketsProcessor implements RpcProcessor Date: Wed, 27 Jul 2022 00:02:25 +0800 Subject: [PATCH 27/35] =?UTF-8?q?=E5=A4=8D=E6=9D=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dataflow/controller/PacketsProcessor.java | 35 ++++++------- .../dataflow/grpc/CollectPackets.java | 1 - .../com/yuandian/dataflow/projo/Response.java | 14 ------ .../dataflow/statemachine/StateMachine.java | 24 ++++----- .../statemachine/StateServerFactory.java | 49 +++++++++---------- .../statemachine/operate/Operate.java | 20 ++++---- .../statemachine/rpc/OperateProcessor.java | 11 ++--- .../statemachine/state/StateFactory.java | 6 +-- .../statemachine/StateMachineTest.java | 2 +- 9 files changed, 68 insertions(+), 94 deletions(-) delete mode 100644 src/main/java/com/yuandian/dataflow/projo/Response.java diff --git a/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java b/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java index e9f6d0a..a35b520 100644 --- a/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java +++ b/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java @@ -55,31 +55,32 @@ public class PacketsProcessor implements RpcProcessor{ + log.info("{} handler request.packets.size(): {}", StateServerFactory.getServerId(), request.packets.size()); + ss.readIndexState((state)->{ - var work = state.getWorkers().get( ss.getCluster().getServerId()); - - - work.setTaskQueueSize( work.getTaskQueueSize() - request.packets.size()); - work.setUpdateAt(Instant.now()); - log.debug("workerState taskQueueSize: {} psize: {} state {}", work.getTaskQueueSize(), request.packets.size(), state.getWorkers().size()); - var op = new Operate(OperateType.PUT); - op.setValue(work); - Operate.CallOperate(op, new OperateClosure() { + var ws = state.getWorkers().get(StateServerFactory.getServerId()); + ws.setTaskQueueSize( ws.getTaskQueueSize() - request.packets.size()); + ws.setUpdateAt(Instant.now()); + + log.debug("workerState taskQueueSize: {} psize: {} state {}", ws.getTaskQueueSize(), request.packets.size(), state.getWorkers().size()); + + Operate.CallOperate(new Operate(OperateType.PUT,ws), new OperateClosure() { @Override public void run(Status status) { - log.info("{}", this.getResponse()); - var resp = new RaftResponse(); - resp.setSuccess(true); - // resp.setRedirect(StateServerFactory.getStateServer().getNode().getLeaderId()); + var resp = this.getResponse(); + if(status.isOk()) { + resp.setSuccess(true); + log.info("{}", resp); + } else { + resp.setSuccess(false); + } + rpcCtx.sendResponse(resp); } - }); }); diff --git a/src/main/java/com/yuandian/dataflow/grpc/CollectPackets.java b/src/main/java/com/yuandian/dataflow/grpc/CollectPackets.java index 0395897..17215ad 100644 --- a/src/main/java/com/yuandian/dataflow/grpc/CollectPackets.java +++ b/src/main/java/com/yuandian/dataflow/grpc/CollectPackets.java @@ -16,7 +16,6 @@ import java.util.stream.Collectors; import com.google.protobuf.*; import com.google.protobuf.util.JsonFormat; import com.yuandian.common.Config; -import com.yuandian.dataflow.projo.Response; import com.yuandian.dataflow.proto.CollectPacketsServerGrpc; diff --git a/src/main/java/com/yuandian/dataflow/projo/Response.java b/src/main/java/com/yuandian/dataflow/projo/Response.java deleted file mode 100644 index 05bfe65..0000000 --- a/src/main/java/com/yuandian/dataflow/projo/Response.java +++ /dev/null @@ -1,14 +0,0 @@ - package com.yuandian.dataflow.projo; - -import org.apache.http.HttpStatus; - -import com.fasterxml.jackson.annotation.JsonProperty; - -public class Response { - @JsonProperty("code") - public org.apache.http.HttpStatus Code; - @JsonProperty("message") - public String Message; - @JsonProperty("data") - public Object Data; -} diff --git a/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java b/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java index 78589c1..3a75347 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java @@ -20,7 +20,7 @@ import com.yuandian.dataflow.statemachine.operate.Operate; import com.yuandian.dataflow.statemachine.operate.Operate.OperateType; import com.yuandian.dataflow.statemachine.rpc.RaftResponse; -import com.yuandian.dataflow.statemachine.rpc.OperateProcessor.RequestOperate; +import com.yuandian.dataflow.statemachine.rpc.OperateProcessor.OperateRequest; import com.yuandian.dataflow.statemachine.state.State; import com.yuandian.dataflow.statemachine.state.StateFactory; import com.yuandian.dataflow.statemachine.state.WorkerState; @@ -147,15 +147,14 @@ public class StateMachine extends StateMachineAdapter { } var ss = StateServerFactory.getStateServer(); - ss.useFsmStateNotLock((state)->{ - var ws = state.getWorkers().get( ss.getCluster().getServerId() ); + ss.readIndexState((state)->{ + var ws = state.getWorkers().get( StateServerFactory.getServerId() ); if(ws == null) { - ws = new WorkerState(ss.getCluster().getServerId()); + ws = new WorkerState(StateServerFactory.getServerId()); // state.getWorkers().put(ss.getCluster().getServerId(), ws); } - Operate op = new Operate(OperateType.PUT); - op.setValue(ws); + Operate op = new Operate(OperateType.PUT, ws); ss.applyOperate(op, new OperateClosure() { @Override public void run(Status status) { @@ -204,10 +203,9 @@ public class StateMachine extends StateMachineAdapter { var ss = StateServerFactory.getStateServer(); var ws = new WorkerState(ss.getCluster().getServerId()); - log.debug("my: {} leader id {}",ss.getCluster().getServerId(), ss.getNode().getLeaderId()); + log.debug("my: {} leader id {}",StateServerFactory.getServerId(), StateServerFactory.getLeaderId()); - var op = new Operate(OperateType.PUT); - op.setValue(ws); + var op = new Operate(OperateType.PUT, ws); Operate.CallOperate(op, new OperateClosure() { @Override @@ -234,13 +232,11 @@ public class StateMachine extends StateMachineAdapter { log.debug("{} {}",ctx, StateServerFactory.getCluster().getServerId()); var ss = StateServerFactory.getStateServer(); - var ws = new WorkerState(ss.getCluster().getServerId()); + var ws = new WorkerState(StateServerFactory.getServerId()); - log.debug("my: {} leader id {}",ss.getCluster().getServerId(), ss.getNode().getLeaderId()); - - var op = new Operate(OperateType.PUT); - op.setValue(ws); + log.debug("my: {} leader id {}",StateServerFactory.getServerId(), StateServerFactory.getLeaderId()); + var op = new Operate(OperateType.PUT, ws); Operate.CallOperate(op, new OperateClosure() { @Override public void run(Status status) { diff --git a/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java b/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java index 69ddde0..66cad64 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java @@ -42,7 +42,7 @@ import com.alipay.sofa.jraft.util.ThreadPoolUtil; import com.yuandian.dataflow.statemachine.operate.Operate; import com.yuandian.dataflow.statemachine.rpc.RaftResponse; -import com.yuandian.dataflow.statemachine.rpc.OperateProcessor.RequestOperate; +import com.yuandian.dataflow.statemachine.rpc.OperateProcessor.OperateRequest; import com.yuandian.dataflow.statemachine.rpc.annotations.ProcessorRaft; import com.yuandian.dataflow.statemachine.state.State; import com.yuandian.dataflow.statemachine.state.WorkerState; @@ -64,9 +64,6 @@ public class StateServerFactory { private static StateServer ss; - private static String myPeerStr; - private static Configuration raftConf; - public static void startStateServer(String peerstr, Configuration conf) throws Exception { if(ss != null) { throw new Exception("重复初始化 InitStateServer"); @@ -75,9 +72,14 @@ public class StateServerFactory { } + public static boolean isLeader() { + return ss.getNode().isLeader() ; + } - + public static PeerId getLeaderId() { + return ss.node.getLeaderId() ; + } public static PeerId getServerId() { return ss.getCluster().getServerId(); @@ -177,21 +179,15 @@ public class StateServerFactory { } } - public void useFsmStateNotLock(Consumer dofunc) { - - - - + public void readIndexState(Consumer dofunc) { getNode().readIndex(BytesUtil.EMPTY_BYTES, new ReadIndexClosure() { - @Override public void run(Status status, long index, byte[] reqCtx) { - var state = ss.fsm.getState(); - dofunc.accept(state); + if( status.isOk()) { + dofunc.accept(ss.fsm.getState()); + } } - } ); - } @@ -204,7 +200,6 @@ public class StateServerFactory { } try { - closure.setValue(op); final Task task = new Task(); task.setData(ByteBuffer.wrap(SerializerManager.getSerializer(SerializerManager.Hessian2).serialize(op))); @@ -242,16 +237,16 @@ public class StateServerFactory { // final StoreEngineOptions opts = new StoreEngineOptions(); // return StoreEngineHelper.createReadIndexExecutor(opts.getReadIndexCoreThreads()); - return ThreadPoolUtil.newBuilder() // - .poolName("ReadIndexPool") // - .enableMetric(true) // - .coreThreads(4) // - .maximumThreads(4) // - .keepAliveSeconds(60L) // - .workQueue(new SynchronousQueue<>()) // - .threadFactory(new NamedThreadFactory("ReadIndexService", true)) // - .rejectedHandler(new ThreadPoolExecutor.AbortPolicy()) // - .build(); + return ThreadPoolUtil.newBuilder() // + .poolName("ReadIndexPool") // + .enableMetric(true) // + .coreThreads(4) // + .maximumThreads(4) // + .keepAliveSeconds(60L) // + .workQueue(new SynchronousQueue<>()) // + .threadFactory(new NamedThreadFactory("ReadIndexService", true)) // + .rejectedHandler(new ThreadPoolExecutor.AbortPolicy()) // + .build(); } } @@ -260,7 +255,7 @@ public class StateServerFactory { public static void main(String[] args) throws InterruptedException, RemotingException { var rpcClient = new BoltRaftRpcFactory().createRpcClient(); rpcClient.init(new CliOptions()); - var resp = rpcClient.invokeSync(new Endpoint("localhost",4441), new RequestOperate(), 5000); + var resp = rpcClient.invokeSync(new Endpoint("localhost",4441), new OperateRequest(), 5000); log.info("{}", resp); } } diff --git a/src/main/java/com/yuandian/dataflow/statemachine/operate/Operate.java b/src/main/java/com/yuandian/dataflow/statemachine/operate/Operate.java index d798fdf..bab840a 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/operate/Operate.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/operate/Operate.java @@ -2,17 +2,14 @@ package com.yuandian.dataflow.statemachine.operate; import java.io.Serializable; -import com.alipay.sofa.jraft.Status; import com.alipay.sofa.jraft.error.RemotingException; import com.alipay.sofa.jraft.rpc.InvokeCallback; -import com.yuandian.dataflow.projo.Response; import com.yuandian.dataflow.statemachine.OperateClosure; import com.yuandian.dataflow.statemachine.StateServerFactory; import com.yuandian.dataflow.statemachine.rpc.OperateProcessor; import com.yuandian.dataflow.statemachine.rpc.RaftResponse; -import com.yuandian.dataflow.statemachine.state.StateFactory; +import com.yuandian.dataflow.statemachine.state.WorkerState; - import lombok.Data; import lombok.var; import lombok.extern.slf4j.Slf4j; @@ -34,8 +31,9 @@ public class Operate implements Serializable { private OperateType type; private Object value; - public Operate(OperateType t) { + public Operate(OperateType t, WorkerState ws) { this.type = t; + this.value = ws; } public T getValue() { @@ -48,17 +46,17 @@ public class Operate implements Serializable { }; public static void CallOperate(Operate op, OperateClosure closure) { - + var ss = StateServerFactory.getStateServer(); - if (StateServerFactory.getStateServer().isLeader()) { + if (StateServerFactory.isLeader()) { ss.applyOperate(op, closure); return; } - var request = new OperateProcessor.RequestOperate(); + var request = new OperateProcessor.OperateRequest(); request.setOperate(op); - var leaderId = StateServerFactory.getStateServer().getNode().getLeaderId(); + var leaderId = StateServerFactory.getLeaderId(); try { ss.getRpcClient().invokeAsync(leaderId.getEndpoint(), request, new InvokeCallback() { @@ -66,7 +64,7 @@ public class Operate implements Serializable { @Override public void complete(Object result, Throwable err) { log.info("{}", result); - var resp = (RaftResponse)result; + var resp = (RaftResponse) result; closure.setResponse(resp); closure.success(resp.getOperate()); } @@ -75,7 +73,7 @@ public class Operate implements Serializable { } catch (InterruptedException | RemotingException e) { // TODO Auto-generated catch block closure.failure("failure", null); - log.info("{}", e.toString()); + log.info("{}", e.toString()); } } diff --git a/src/main/java/com/yuandian/dataflow/statemachine/rpc/OperateProcessor.java b/src/main/java/com/yuandian/dataflow/statemachine/rpc/OperateProcessor.java index 6c96a28..b8b4c8a 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/rpc/OperateProcessor.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/rpc/OperateProcessor.java @@ -40,19 +40,18 @@ import lombok.extern.slf4j.Slf4j; */ @Slf4j @ProcessorRaft -public class OperateProcessor implements RpcProcessor { +public class OperateProcessor implements RpcProcessor { /** - * 同步状态时, 需要用的结构类. 新增的状态可以在 State结构里添加 + * 同步状态时, 需要用的结构类. 新增的状态可以在 Operate结构里添加 * * @author eson *2022年7月11日-16:01:07 */ - @Getter @Setter @ToString - public static class RequestOperate implements Serializable { + public static class OperateRequest implements Serializable { private static final long serialVersionUID = 1L; @@ -61,7 +60,7 @@ public class OperateProcessor implements RpcProcessor { + ss.readIndexState((state) -> { alivePeers.forEach((peer) -> { WorkerState ws = state.getWorkers().get(peer); if (ws != null) { @@ -89,8 +89,8 @@ public class StateFactory { request.getPackets().add(p); } - var op = new Operate(OperateType.PUT); - op.setValue(ws); + var op = new Operate(OperateType.PUT, ws); + Operate.CallOperate(op, new OperateClosure() { @Override public void run(Status status) { diff --git a/src/test/java/com/yuandian/dataflow/statemachine/StateMachineTest.java b/src/test/java/com/yuandian/dataflow/statemachine/StateMachineTest.java index eed9347..5e117a0 100644 --- a/src/test/java/com/yuandian/dataflow/statemachine/StateMachineTest.java +++ b/src/test/java/com/yuandian/dataflow/statemachine/StateMachineTest.java @@ -10,7 +10,7 @@ import com.alipay.sofa.jraft.rpc.InvokeCallback; import com.alipay.sofa.jraft.rpc.impl.BoltRaftRpcFactory; import com.alipay.sofa.jraft.util.Endpoint; import com.yuandian.dataflow.statemachine.rpc.RaftResponse; -import com.yuandian.dataflow.statemachine.rpc.OperateProcessor.RequestOperate; +import com.yuandian.dataflow.statemachine.rpc.OperateProcessor.OperateRequest; import com.yuandian.dataflow.statemachine.state.State; import lombok.var; From 4fa9035983f5d185908e42870b64174f39b404e0 Mon Sep 17 00:00:00 2001 From: huangsimin <474420502@qq.com> Date: Wed, 27 Jul 2022 12:37:32 +0800 Subject: [PATCH 28/35] =?UTF-8?q?=E6=B7=BB=E5=8A=A0mongodb=E4=BE=8B?= =?UTF-8?q?=E5=AD=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 16 +- .../dataflow/controller/PacketsProcessor.java | 55 +++---- .../java/com/yuandian/dataflow/projo/Doc.java | 59 +++++-- .../dataflow/statemachine/MasterFactory.java | 124 +++++++++++++++ .../dataflow/statemachine/StateMachine.java | 77 +++++---- .../statemachine/StateServerFactory.java | 20 ++- .../GenericClosure.java} | 21 ++- .../statemachine/closure/StateClosure.java | 5 + .../statemachine/operate/Operate.java | 19 ++- .../statemachine/rpc/OperateProcessor.java | 4 +- .../statemachine/rpc/RaftResponse.java | 4 +- .../statemachine/state/StateFactory.java | 146 ------------------ .../java/com/yuandian/dataflow/AppTest.java | 107 ++++++------- .../com/yuandian/dataflow/MongodbTest.java | 122 +++++++-------- 14 files changed, 406 insertions(+), 373 deletions(-) create mode 100644 src/main/java/com/yuandian/dataflow/statemachine/MasterFactory.java rename src/main/java/com/yuandian/dataflow/statemachine/{OperateClosure.java => closure/GenericClosure.java} (67%) create mode 100644 src/main/java/com/yuandian/dataflow/statemachine/closure/StateClosure.java delete mode 100644 src/main/java/com/yuandian/dataflow/statemachine/state/StateFactory.java diff --git a/pom.xml b/pom.xml index d35d4da..480ac89 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ 1.7.36 1.3.11 2.7.1 - 3.12.11 + 4.7.0 2.1.0 1.30 1.2.11 @@ -83,11 +83,15 @@ - - org.mongodb - mongo-java-driver - ${mongo.driver.version} - + + + + + org.mongodb + mongodb-driver-sync + ${mongo.driver.version} + + diff --git a/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java b/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java index a35b520..14edd27 100644 --- a/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java +++ b/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java @@ -18,9 +18,9 @@ import com.alipay.sofa.jraft.rpc.RpcContext; import com.alipay.sofa.jraft.rpc.RpcProcessor; import com.google.protobuf.Any; import com.yuandian.dataflow.statemachine.StateServerFactory; +import com.yuandian.dataflow.statemachine.closure.GenericClosure; import com.yuandian.dataflow.statemachine.operate.Operate; import com.yuandian.dataflow.statemachine.operate.Operate.OperateType; -import com.yuandian.dataflow.statemachine.OperateClosure; import com.yuandian.dataflow.statemachine.rpc.RaftResponse; import com.yuandian.dataflow.statemachine.rpc.OperateProcessor; @@ -52,37 +52,38 @@ public class PacketsProcessor implements RpcProcessor{ - - var ws = state.getWorkers().get(StateServerFactory.getServerId()); - ws.setTaskQueueSize( ws.getTaskQueueSize() - request.packets.size()); - ws.setUpdateAt(Instant.now()); + ss.readIndexState( new GenericClosure() { - log.debug("workerState taskQueueSize: {} psize: {} state {}", ws.getTaskQueueSize(), request.packets.size(), state.getWorkers().size()); - - Operate.CallOperate(new Operate(OperateType.PUT,ws), new OperateClosure() { - @Override - public void run(Status status) { - var resp = this.getResponse(); - if(status.isOk()) { - resp.setSuccess(true); - log.info("{}", resp); - } else { - resp.setSuccess(false); + @Override + public void run(Status status) { + + var state = this.getValue(); + var ws = state.getWorkers().get(StateServerFactory.getServerId()); + ws.setTaskQueueSize( ws.getTaskQueueSize() - request.packets.size()); + ws.setUpdateAt(Instant.now()); + + log.debug("workerState taskQueueSize: {} psize: {} state {}", ws.getTaskQueueSize(), request.packets.size(), state.getWorkers().size()); + + Operate.CallOperate(new Operate(OperateType.PUT,ws), new GenericClosure() { + @Override + public void run(Status status) { + var resp = this.getResponse(); + resp.setMsg(rpcCtx.getRemoteAddress()); + if(status.isOk()) { + resp.setSuccess(true); + log.info("{}", resp); + } else { + resp.setSuccess(false); + } + rpcCtx.sendResponse(resp); } - - rpcCtx.sendResponse(resp); - } - }); - }); + }); + } + + } ); } diff --git a/src/main/java/com/yuandian/dataflow/projo/Doc.java b/src/main/java/com/yuandian/dataflow/projo/Doc.java index e1389a1..3d073fd 100644 --- a/src/main/java/com/yuandian/dataflow/projo/Doc.java +++ b/src/main/java/com/yuandian/dataflow/projo/Doc.java @@ -4,33 +4,66 @@ package com.yuandian.dataflow.projo; import java.time.LocalDateTime; +import org.bson.BsonDocument; +import org.bson.BsonInt64; +import org.bson.BsonValue; import org.bson.Document; +import org.bson.codecs.configuration.CodecProvider; +import org.bson.codecs.configuration.CodecRegistry; +import org.bson.codecs.pojo.PojoCodecProvider; +import static com.mongodb.MongoClientSettings.getDefaultCodecRegistry; +import static org.bson.codecs.configuration.CodecRegistries.fromProviders; +import static org.bson.codecs.configuration.CodecRegistries.fromRegistries; import org.bson.codecs.pojo.annotations.BsonProperty; +import org.bson.conversions.Bson; +import org.bson.types.ObjectId; import com.fasterxml.jackson.annotation.JsonProperty; +import com.mongodb.client.FindIterable; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoDatabase; +import com.mongodb.client.model.Filters; import lombok.Getter; import lombok.Setter; +import lombok.ToString; +import lombok.extern.slf4j.Slf4j; +@Slf4j @Setter @Getter -public final class Doc extends Document { +@ToString +public final class Doc { - @JsonProperty("code") - @BsonProperty("code") - public int Code ; + @BsonProperty("retryPackets") + public int retryPackets ; - @JsonProperty("ts") - @BsonProperty("ts") - public LocalDateTime TS; + @BsonProperty("serverResponseTime") + public int serverResponseTime ; - @JsonProperty("desc") - @BsonProperty("desc") - public String Desc; + @BsonProperty("requestBytes") + public int requestBytes ; - @JsonProperty("data") - @BsonProperty("data") - public Document Data; + @BsonProperty("businessName") + public String businessName ; + + @BsonProperty("responseIp") + public int responseIp ; + + + + + public static void main(String[] args) { + MongoClient mgo = MongoClients.create("mongodb://yuandian:yuandian123@192.168.1.113:27017"); + CodecProvider pojoCodecProvider = PojoCodecProvider.builder().register("com.yuandian.dataflow.projo").build(); + CodecRegistry pojoCodecRegistry = fromRegistries(getDefaultCodecRegistry(),fromProviders(pojoCodecProvider)); + + MongoDatabase oriDatabase = mgo.getDatabase("ori-database").withCodecRegistry(pojoCodecRegistry); + MongoCollection db = oriDatabase.getCollection("network_performace_flow_2022072400", Doc.class); + log.debug("{}", db.countDocuments( new BsonDocument("serverResponseTime", new BsonDocument("$gt", new BsonInt64(2083478517) )) )); + } } diff --git a/src/main/java/com/yuandian/dataflow/statemachine/MasterFactory.java b/src/main/java/com/yuandian/dataflow/statemachine/MasterFactory.java new file mode 100644 index 0000000..9263085 --- /dev/null +++ b/src/main/java/com/yuandian/dataflow/statemachine/MasterFactory.java @@ -0,0 +1,124 @@ +/** + * description + * + * @author eson + *2022年7月20日-10:00:05 + */ +package com.yuandian.dataflow.statemachine; + +import java.time.Instant; + +import com.alipay.sofa.jraft.Status; +import com.alipay.sofa.jraft.error.RemotingException; +import com.alipay.sofa.jraft.rpc.InvokeCallback; +import com.google.protobuf.Any; +import com.yuandian.dataflow.controller.PacketsProcessor.PacketsRequest; +import com.yuandian.dataflow.proto.msgtype.BacktrackingFlowOuterClass; +import com.yuandian.dataflow.statemachine.closure.GenericClosure; +import com.yuandian.dataflow.statemachine.operate.Operate; +import com.yuandian.dataflow.statemachine.operate.Operate.OperateType; +import com.yuandian.dataflow.statemachine.state.State; +import com.yuandian.dataflow.statemachine.state.WorkerState; + +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import lombok.var; +import lombok.extern.slf4j.Slf4j; + +/** + * Master主线程, 用于接收packets + * + * @author eson + * 2022年7月20日-10:00:05 + */ +@Slf4j +@Getter +@Setter +@ToString +public class MasterFactory { + + public static final int MAX_TASKS = 100; + + public static Thread masterExecute = new Thread(new Runnable() { + @Override + public void run() { + try { + while (true) { + log.debug("master execute {}", StateServerFactory.getServerId()); + var alivePeers = StateServerFactory.getRaftNode().listAlivePeers(); + log.debug("master execute {}", StateServerFactory.getRaftNode().listAlivePeers()); + if (alivePeers != null) { + + var ss = StateServerFactory.getStateServer(); + // 读一致性 + ss.readIndexState( new GenericClosure() { + + @Override + public void run(Status status) { + var state = this.getValue(); + + alivePeers.forEach((peer) -> { + WorkerState ws = state.getWorkers().get(peer); + if (ws != null) { + // + var canDealTasks = MAX_TASKS - ws.getTaskQueueSize(); + log.debug("cap :{} peer: {}", canDealTasks, peer); + if (canDealTasks <= 0) { + return; + } + ws.setUpdateAt(Instant.now()); + ws.setTaskQueueSize(MAX_TASKS); + + var request = new PacketsRequest(); + for (int i = 0; i < canDealTasks; i++) { + var p = Any.pack(BacktrackingFlowOuterClass.BacktrackingFlow + .newBuilder() + .setTableId(10086) + .build()); + request.getPackets().add(p); + } + + Operate.CallOperate(new Operate(OperateType.PUT, ws), new GenericClosure() { + @Override + public void run(Status status) { + + log.info("{}", status); + try { + ss.getRpcClient().invokeAsync(peer.getEndpoint(), request, + new InvokeCallback() { + @Override + public void complete(Object result, Throwable err) { + log.info("PacketsRequest: {}", result); + } + }, 5000); + } catch (InterruptedException | RemotingException e) { + log.info("error send packets {}", e.toString()); + } + } + }); + } + }); + } + + } ); + } + + Thread.sleep(5000); + } + } catch (InterruptedException e) { + log.info("{}", e.toString()); + } + + } + }); + + public static Thread getMasterExecute() { + return masterExecute; + } + + public static void Init() { + + } + +} diff --git a/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java b/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java index 3a75347..cf3b6c1 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java @@ -16,13 +16,13 @@ import com.alipay.sofa.jraft.error.RaftException; import com.alipay.sofa.jraft.error.RemotingException; import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader; import com.alipay.sofa.jraft.storage.snapshot.SnapshotWriter; +import com.yuandian.dataflow.statemachine.closure.GenericClosure; import com.yuandian.dataflow.statemachine.operate.Operate; import com.yuandian.dataflow.statemachine.operate.Operate.OperateType; import com.yuandian.dataflow.statemachine.rpc.RaftResponse; import com.yuandian.dataflow.statemachine.rpc.OperateProcessor.OperateRequest; import com.yuandian.dataflow.statemachine.state.State; -import com.yuandian.dataflow.statemachine.state.StateFactory; import com.yuandian.dataflow.statemachine.state.WorkerState; import lombok.var; @@ -66,12 +66,11 @@ public class StateMachine extends StateMachineAdapter { while (iter.hasNext()) { Operate op = null; - OperateClosure closure = null; + GenericClosure closure = null; if (iter.done() != null) { // This task is applied by this node, get value from closure to avoid additional // parsing. - closure = (OperateClosure)iter.done(); // 只支持单一个State. 全状态机只支持一种提交 - // log.info("SyncDataClosure(done) taskState:{} leaderTerm:{} {}",this.state, this.leaderTerm, closure); + closure = (GenericClosure)iter.done(); // 只支持单一个State. 全状态机只支持一种提交 op = closure.getValue(); } else { @@ -79,11 +78,7 @@ public class StateMachine extends StateMachineAdapter { final ByteBuffer data = iter.getData(); try { - op = SerializerManager.getSerializer(SerializerManager.Hessian2).deserialize( - data.array(),Operate.class.getName()); - // log.info("SyncDataClosure(null) taskState:{} leaderTerm:{}", state, this.leaderTerm); - - + op = SerializerManager.getSerializer(SerializerManager.Hessian2).deserialize(data.array(),Operate.class.getName()); } catch (CodecException e) { log.info("{}", e.toString()); } @@ -101,8 +96,6 @@ public class StateMachine extends StateMachineAdapter { } break; case REMOVE: - - if(closure != null) { closure.success(op); closure.run(Status.OK()); @@ -139,33 +132,39 @@ public class StateMachine extends StateMachineAdapter { @Override public void onLeaderStart(final long term) { - log.debug("onLeaderStart {}", StateServerFactory.getCluster().getServerId()); + log.debug("onLeaderStart {}", StateServerFactory.getServerId()); this.leaderTerm.set(term); - if(StateFactory.getMasterExecute().isAlive()) { - StateFactory.getMasterExecute().interrupt(); + // 判断是否Master线程还在跑, 如果存在则中断 + if(MasterFactory.getMasterExecute().isAlive()) { + MasterFactory.getMasterExecute().interrupt(); } var ss = StateServerFactory.getStateServer(); - ss.readIndexState((state)->{ - var ws = state.getWorkers().get( StateServerFactory.getServerId() ); - if(ws == null) { - ws = new WorkerState(StateServerFactory.getServerId()); - // state.getWorkers().put(ss.getCluster().getServerId(), ws); - } + ss.readIndexState( new GenericClosure() { - Operate op = new Operate(OperateType.PUT, ws); - ss.applyOperate(op, new OperateClosure() { - @Override - public void run(Status status) { - log.debug("master update workerstate: {}", status); + @Override + public void run(Status status) { + + var ws = state.getWorkers().get( StateServerFactory.getServerId() ); + if(ws == null) { + ws = new WorkerState(StateServerFactory.getServerId()); } - }); - + + Operate op = new Operate(OperateType.PUT, ws); + ss.applyOperate(op, new GenericClosure() { + @Override + public void run(Status status) { + log.debug("master update workerstate: {}", status); + } + }); + } + }); - StateFactory.getMasterExecute().start(); - + // 当成为master时候 必须启动 + MasterFactory.getMasterExecute().start(); + super.onLeaderStart(term); } @@ -175,8 +174,9 @@ public class StateMachine extends StateMachineAdapter { this.leaderTerm.set(-1); super.onLeaderStop(status); - if(StateFactory.getMasterExecute().isAlive()) { - StateFactory.getMasterExecute().interrupt(); + // 判断是否Master线程还在跑, 如果存在则中断 + if(MasterFactory.getMasterExecute().isAlive()) { + MasterFactory.getMasterExecute().interrupt(); } @@ -196,18 +196,17 @@ public class StateMachine extends StateMachineAdapter { log.debug("[onStartFollowing] {} {}", ctx, StateServerFactory.getCluster().getServerId()); try { - if(StateFactory.getMasterExecute().isAlive()) { - StateFactory.getMasterExecute().interrupt(); + // 判断是否Master线程还在跑, 如果存在则中断 + if(MasterFactory.getMasterExecute().isAlive()) { + MasterFactory.getMasterExecute().interrupt(); } - - var ss = StateServerFactory.getStateServer(); - var ws = new WorkerState(ss.getCluster().getServerId()); - + + var ws = new WorkerState(StateServerFactory.getServerId()); log.debug("my: {} leader id {}",StateServerFactory.getServerId(), StateServerFactory.getLeaderId()); var op = new Operate(OperateType.PUT, ws); - Operate.CallOperate(op, new OperateClosure() { + Operate.CallOperate(op, new GenericClosure() { @Override public void run(Status status) { log.info("{} {}", status, this.getResponse()); @@ -237,7 +236,7 @@ public class StateMachine extends StateMachineAdapter { log.debug("my: {} leader id {}",StateServerFactory.getServerId(), StateServerFactory.getLeaderId()); var op = new Operate(OperateType.PUT, ws); - Operate.CallOperate(op, new OperateClosure() { + Operate.CallOperate(op, new GenericClosure() { @Override public void run(Status status) { log.info("{} {}", status, this.getResponse()); diff --git a/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java b/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java index 66cad64..c5d724a 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java @@ -39,6 +39,7 @@ import com.alipay.sofa.jraft.rpc.impl.BoltRaftRpcFactory; import com.alipay.sofa.jraft.util.BytesUtil; import com.alipay.sofa.jraft.util.Endpoint; import com.alipay.sofa.jraft.util.ThreadPoolUtil; +import com.yuandian.dataflow.statemachine.closure.GenericClosure; import com.yuandian.dataflow.statemachine.operate.Operate; import com.yuandian.dataflow.statemachine.rpc.RaftResponse; @@ -73,7 +74,7 @@ public class StateServerFactory { public static boolean isLeader() { - return ss.getNode().isLeader() ; + return ss.node.isLeader() ; } @@ -82,11 +83,15 @@ public class StateServerFactory { } public static PeerId getServerId() { - return ss.getCluster().getServerId(); + return ss.cluster.getServerId(); } public static Node getNode() { - return ss.getNode() ; + return ss.node ; + } + + public static Node getRaftNode() { + return ss.cluster.getRaftNode() ; } public static RpcClient getRpcClient() { @@ -179,12 +184,13 @@ public class StateServerFactory { } } - public void readIndexState(Consumer dofunc) { + public void readIndexState(GenericClosure closure) { getNode().readIndex(BytesUtil.EMPTY_BYTES, new ReadIndexClosure() { @Override public void run(Status status, long index, byte[] reqCtx) { if( status.isOk()) { - dofunc.accept(ss.fsm.getState()); + closure.success(ss.fsm.getState()); + closure.run(status); } } } ); @@ -192,7 +198,7 @@ public class StateServerFactory { - public void applyOperate(Operate op, OperateClosure closure) { + public void applyOperate(Operate op, GenericClosure closure) { // 所有的提交都必须再leader进行 if (!ss.isLeader()) { ss.handlerNotLeaderError(closure); @@ -227,7 +233,7 @@ public class StateServerFactory { return response; } - public void handlerNotLeaderError(final OperateClosure closure) { + public void handlerNotLeaderError(final GenericClosure closure) { closure.failure("Not leader.", redirect().getRedirect()); closure.run(new Status(RaftError.EPERM, "Not leader")); } diff --git a/src/main/java/com/yuandian/dataflow/statemachine/OperateClosure.java b/src/main/java/com/yuandian/dataflow/statemachine/closure/GenericClosure.java similarity index 67% rename from src/main/java/com/yuandian/dataflow/statemachine/OperateClosure.java rename to src/main/java/com/yuandian/dataflow/statemachine/closure/GenericClosure.java index 69a10e7..002bed1 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/OperateClosure.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/closure/GenericClosure.java @@ -1,4 +1,4 @@ -package com.yuandian.dataflow.statemachine; +package com.yuandian.dataflow.statemachine.closure; import com.alipay.sofa.jraft.Closure; import com.alipay.sofa.jraft.Status; @@ -19,31 +19,28 @@ import org.slf4j.LoggerFactory; @Getter @Setter @ToString -public abstract class OperateClosure implements Closure { +public abstract class GenericClosure implements Closure { // 状态机的统一响应 - private RaftResponse response; + private RaftResponse response; // 代表任务状态 - private Operate value; + private T value; - - - - public OperateClosure() { + public GenericClosure() { } public void failure(final String errorMsg, final PeerId redirect) { - final RaftResponse response = new RaftResponse(); + final RaftResponse response = new RaftResponse(); response.setSuccess(false); response.setMsg(errorMsg); response.setRedirect(redirect); setResponse(response); } - public void success(final Operate value) { - final RaftResponse response = new RaftResponse(); - response.setOperate(value); + public void success(final T value) { + final RaftResponse response = new RaftResponse(); + response.setValue(value); response.setSuccess(true); setResponse(response); } diff --git a/src/main/java/com/yuandian/dataflow/statemachine/closure/StateClosure.java b/src/main/java/com/yuandian/dataflow/statemachine/closure/StateClosure.java new file mode 100644 index 0000000..e49a805 --- /dev/null +++ b/src/main/java/com/yuandian/dataflow/statemachine/closure/StateClosure.java @@ -0,0 +1,5 @@ +package com.yuandian.dataflow.statemachine.closure; + +public class StateClosure { + +} diff --git a/src/main/java/com/yuandian/dataflow/statemachine/operate/Operate.java b/src/main/java/com/yuandian/dataflow/statemachine/operate/Operate.java index bab840a..0194b1a 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/operate/Operate.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/operate/Operate.java @@ -4,8 +4,8 @@ import java.io.Serializable; import com.alipay.sofa.jraft.error.RemotingException; import com.alipay.sofa.jraft.rpc.InvokeCallback; -import com.yuandian.dataflow.statemachine.OperateClosure; import com.yuandian.dataflow.statemachine.StateServerFactory; +import com.yuandian.dataflow.statemachine.closure.GenericClosure; import com.yuandian.dataflow.statemachine.rpc.OperateProcessor; import com.yuandian.dataflow.statemachine.rpc.RaftResponse; import com.yuandian.dataflow.statemachine.state.WorkerState; @@ -45,14 +45,21 @@ public class Operate implements Serializable { return; }; - public static void CallOperate(Operate op, OperateClosure closure) { + /** + * 调用操作设置 + * @param op 传入的操作类 + * @param closure 回调函数. Operate为返回值 + */ + public static void CallOperate(Operate op, GenericClosure closure) { var ss = StateServerFactory.getStateServer(); - if (StateServerFactory.isLeader()) { + // 如果是leader 就直接提交 + if (StateServerFactory.isLeader()) { ss.applyOperate(op, closure); return; } + // 非leader 转发请求 统一有leader处理 var request = new OperateProcessor.OperateRequest(); request.setOperate(op); @@ -64,14 +71,14 @@ public class Operate implements Serializable { @Override public void complete(Object result, Throwable err) { log.info("{}", result); - var resp = (RaftResponse) result; + //TODO: 解决回调的次序问题 + var resp = (RaftResponse) result; closure.setResponse(resp); - closure.success(resp.getOperate()); + closure.success(resp.getValue()); } }, 5000); } catch (InterruptedException | RemotingException e) { - // TODO Auto-generated catch block closure.failure("failure", null); log.info("{}", e.toString()); } diff --git a/src/main/java/com/yuandian/dataflow/statemachine/rpc/OperateProcessor.java b/src/main/java/com/yuandian/dataflow/statemachine/rpc/OperateProcessor.java index b8b4c8a..96a825a 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/rpc/OperateProcessor.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/rpc/OperateProcessor.java @@ -17,8 +17,8 @@ import com.alipay.sofa.jraft.error.RaftError; import com.alipay.sofa.jraft.rpc.RpcContext; import com.alipay.sofa.jraft.rpc.RpcProcessor; import com.yuandian.dataflow.statemachine.StateServerFactory; -import com.yuandian.dataflow.statemachine.OperateClosure; import com.yuandian.dataflow.statemachine.StateServerFactory.StateServer; +import com.yuandian.dataflow.statemachine.closure.GenericClosure; import com.yuandian.dataflow.statemachine.operate.Operate; import com.yuandian.dataflow.statemachine.rpc.annotations.ProcessorRaft; import com.yuandian.dataflow.statemachine.state.State; @@ -64,7 +64,7 @@ public class OperateProcessor implements RpcProcessor implements Serializable { private static final long serialVersionUID = 1L; - private Operate operate; + private T value; private boolean success; diff --git a/src/main/java/com/yuandian/dataflow/statemachine/state/StateFactory.java b/src/main/java/com/yuandian/dataflow/statemachine/state/StateFactory.java deleted file mode 100644 index 65e5cb2..0000000 --- a/src/main/java/com/yuandian/dataflow/statemachine/state/StateFactory.java +++ /dev/null @@ -1,146 +0,0 @@ -/** - * description - * - * @author eson - *2022年7月20日-10:00:05 - */ -package com.yuandian.dataflow.statemachine.state; - -import java.time.Instant; -import java.util.ArrayList; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import com.alibaba.nacos.shaded.io.grpc.netty.shaded.io.netty.handler.codec.http.HttpContentEncoder.Result; -import com.alipay.sofa.jraft.Status; -import com.alipay.sofa.jraft.entity.PeerId; -import com.alipay.sofa.jraft.error.RemotingException; -import com.alipay.sofa.jraft.rpc.InvokeCallback; -import com.google.protobuf.Any; -import com.yuandian.dataflow.controller.PacketsProcessor.PacketsRequest; -import com.yuandian.dataflow.proto.msgtype.BacktrackingFlowOuterClass; -import com.yuandian.dataflow.statemachine.StateServerFactory; -import com.yuandian.dataflow.statemachine.operate.Operate; -import com.yuandian.dataflow.statemachine.operate.Operate.OperateType; -import com.yuandian.dataflow.statemachine.OperateClosure; - -import lombok.Getter; -import lombok.Setter; -import lombok.ToString; -import lombok.var; -import lombok.extern.slf4j.Slf4j; - -/** - * description - * - * @author eson - * 2022年7月20日-10:00:05 - */ -@Slf4j -@Getter -@Setter -@ToString -public class StateFactory { - - @Getter - @Setter - @ToString - public static class PeerIdCap { - private PeerId peer; - private long cap; - - public PeerIdCap(PeerId pid, long cap) { - this.peer = pid; - this.cap = cap; - } - } - - public static Thread masterExecute = new Thread(new Runnable() { - @Override - public void run() { - try { - while (true) { - log.debug("master execute {}", StateServerFactory.getServerId()); - var alivePeers = StateServerFactory.getCluster().getRaftNode().listAlivePeers(); - log.debug("master execute {}", StateServerFactory.getCluster().getRaftNode().listAlivePeers()); - if (alivePeers != null) { - - var ss = StateServerFactory.getStateServer(); - // var state = ss.getFsm().getState(); - - ss.readIndexState((state) -> { - alivePeers.forEach((peer) -> { - WorkerState ws = state.getWorkers().get(peer); - if (ws != null) { - var cap = 100 - ws.getTaskQueueSize(); - log.debug("cap :{} peer: {}", cap, peer); - if (cap <= 0) { - return; - } - ws.setUpdateAt(Instant.now()); - ws.setTaskQueueSize(100); - - var request = new PacketsRequest(); - for (int i = 0; i < cap; i++) { - var p = Any.pack( - BacktrackingFlowOuterClass.BacktrackingFlow.newBuilder() - .setTableId(10086) - .build()); - request.getPackets().add(p); - } - - var op = new Operate(OperateType.PUT, ws); - - Operate.CallOperate(op, new OperateClosure() { - @Override - public void run(Status status) { - - log.info("{}", status); - - try { - ss.getRpcClient().invokeAsync(peer.getEndpoint(), - request, new InvokeCallback() { - - @Override - public void complete(Object result, Throwable err) { - log.info("{}", result); - } - - }, 5000); - } catch (InterruptedException | RemotingException e) { - log.info("error send packets {}", e.toString()); - } - } - }); - - } - }); - - }); - - // ss.applyState(state, new SyncClosure() { - // public void run(Status status) { - // log.debug("{}", status); - // }; - // }); - - } - - Thread.sleep(5000); - } - } catch (InterruptedException e) { - log.info("{}", e.toString()); - } - - } - }); - - public static Thread getMasterExecute() { - return masterExecute; - } - - public static void Init() { - - } - -} diff --git a/src/test/java/com/yuandian/dataflow/AppTest.java b/src/test/java/com/yuandian/dataflow/AppTest.java index 1117839..88cdb2f 100644 --- a/src/test/java/com/yuandian/dataflow/AppTest.java +++ b/src/test/java/com/yuandian/dataflow/AppTest.java @@ -23,7 +23,8 @@ import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; // import org.springframework.expression.spel.ast.FunctionReference; -import com.mongodb.MongoClient; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; import com.mongodb.client.model.InsertManyOptions; import com.yuandian.dataflow.projo.Doc; @@ -122,75 +123,77 @@ public class AppTest { } } - @Test - public void Mongodb() throws InterruptedException { + // @Test + // public void Mongodb() throws InterruptedException { - ArrayList execs = new ArrayList<>(); + // ArrayList execs = new ArrayList<>(); - final Metric metric = new Metric(); - metric.start(); - for (int c = 0; c < 10; c++) { - Thread exec = new Thread(() -> { + // final Metric metric = new Metric(); + // metric.start(); + // for (int c = 0; c < 10; c++) { + // Thread exec = new Thread(() -> { - @Cleanup - MongoClient mgo = new MongoClient("localhost", 27017); + // @Cleanup + // MongoClient mgo = MongoClients.create("mongodb://yuandian:yuandian123@192.168.1.113:27017"); + + - log.info("msg"); + // log.info("msg"); - long LoopNumber = 5; - long BatchSize = 20000; + // long LoopNumber = 5; + // long BatchSize = 20000; - var db = mgo.getDatabase("yuandian"); - var cltdoc = db.getCollection("doc"); + // var db = mgo.getDatabase("yuandian"); + // var cltdoc = db.getCollection("doc"); - for (int n = 0; n < LoopNumber; n++) { + // for (int n = 0; n < LoopNumber; n++) { - metric.push(() -> { + // metric.push(() -> { - List documents = new ArrayList<>(); - Random r = new Random(); + // List documents = new ArrayList<>(); + // Random r = new Random(); - for (int i = 0; i < BatchSize; i++) { + // for (int i = 0; i < BatchSize; i++) { - var doc = new Doc(); - var datadoc = new Document(); + // var doc = new Doc(); + // var datadoc = new Document(); - doc.append("code", r.nextInt(100)); - doc.append("desc", "desc"); - doc.append("ts", Instant.now()); + // doc.append("code", r.nextInt(100)); + // doc.append("desc", "desc"); + // doc.append("ts", Instant.now()); - for (int ii = 0; ii < 24; ii++) { - UUID uid = UUID.randomUUID(); - datadoc - .append(uid.toString(), uid.toString()); - } + // for (int ii = 0; ii < 24; ii++) { + // UUID uid = UUID.randomUUID(); + // datadoc + // .append(uid.toString(), uid.toString()); + // } - doc.append("data", datadoc); - documents.add(doc); - } + // doc.append("data", datadoc); + // documents.add(doc); + // } - var opt = new InsertManyOptions(); - cltdoc.insertMany(documents, opt); - return BatchSize; - }); - } - }); - exec.start(); - execs.add(exec); - } - ; + // var opt = new InsertManyOptions(); + // cltdoc.insertMany(documents, opt); + // return BatchSize; + // }); + // } + // }); + // exec.start(); + // execs.add(exec); + // } + // ; - execs.forEach((e) -> { - try { - e.join(); - } catch (InterruptedException e1) { - e1.printStackTrace(); - } - }); + // execs.forEach((e) -> { + // try { + // e.join(); + // } catch (InterruptedException e1) { + // e1.printStackTrace(); + // } + // }); - metric.close(); + // metric.close(); - } + // } diff --git a/src/test/java/com/yuandian/dataflow/MongodbTest.java b/src/test/java/com/yuandian/dataflow/MongodbTest.java index 7822bd7..424ed9e 100644 --- a/src/test/java/com/yuandian/dataflow/MongodbTest.java +++ b/src/test/java/com/yuandian/dataflow/MongodbTest.java @@ -1,6 +1,6 @@ package com.yuandian.dataflow; -import com.mongodb.MongoClient; + import com.mongodb.MongoCredential; import com.mongodb.ServerAddress; import com.mongodb.client.MongoCollection; @@ -18,73 +18,73 @@ import java.util.List; public class MongodbTest { public static void insertMsgToMongoDB(T obj) { - try { - ServerAddress serverAddress = new ServerAddress("192.168.1.113", 27017); - List addrs = new ArrayList<>(); - addrs.add(serverAddress); + // try { + // ServerAddress serverAddress = new ServerAddress("192.168.1.113", 27017); + // List addrs = new ArrayList<>(); + // addrs.add(serverAddress); - MongoCredential credential = MongoCredential.createScramSha1Credential("yd-rw", "admin", "yuandian123".toCharArray()); - List credentials = new ArrayList<>(); - credentials.add(credential); + // MongoCredential credential = MongoCredential.createScramSha1Credential("yd-rw", "admin", "yuandian123".toCharArray()); + // List credentials = new ArrayList<>(); + // credentials.add(credential); - MongoClient mongoClient = new MongoClient(addrs, credentials); + // MongoClient mongoClient = new MongoClient(addrs, credentials); - MongoDatabase db = mongoClient.getDatabase("yd-base"); + // MongoDatabase db = mongoClient.getDatabase("yd-base"); - // todo 修改名字 - MongoCollection collection = db.getCollection("lxy-test"); + // // todo 修改名字 + // MongoCollection collection = db.getCollection("lxy-test"); - collection.insertOne(obj2Doc(obj)); + // collection.insertOne(obj2Doc(obj)); - System.err.println("insert success"); - } catch (Exception e) { - log.info("{}", e.toString()); - } - } + // System.err.println("insert success"); + // } catch (Exception e) { + // log.info("{}", e.toString()); + // } + // } - public static Document obj2Doc(T obj) throws Exception { - Document doc = new Document(); - Field[] fields = obj.getClass().getDeclaredFields(); - for (Field field : fields) { - String varName = field.getName(); - boolean accessFlag = field.isAccessible(); - if (!accessFlag) { - field.setAccessible(true); - } - Object param = field.get(obj); - if (param == null) { - continue; - } else if (param instanceof Integer) { - int value = ((Integer) param).intValue(); - doc.put(varName, value); - } else if (param instanceof String) { - String value = (String) param; - doc.put(varName, value); - } else if (param instanceof Double) { - double value = ((Double) param).doubleValue(); - doc.put(varName, value); - } else if (param instanceof Float) { - float value = ((Float) param).floatValue(); - doc.put(varName, value); - } else if (param instanceof Long) { - long value = ((Long) param).longValue(); - doc.put(varName, value); - } else if (param instanceof Boolean) { - boolean value = ((Boolean) param).booleanValue(); - doc.put(varName, value); - } - field.setAccessible(accessFlag); - } - return doc; - } + // public static Document obj2Doc(T obj) throws Exception { + // Document doc = new Document(); + // Field[] fields = obj.getClass().getDeclaredFields(); + // for (Field field : fields) { + // String varName = field.getName(); + // boolean accessFlag = field.isAccessible(); + // if (!accessFlag) { + // field.setAccessible(true); + // } + // Object param = field.get(obj); + // if (param == null) { + // continue; + // } else if (param instanceof Integer) { + // int value = ((Integer) param).intValue(); + // doc.put(varName, value); + // } else if (param instanceof String) { + // String value = (String) param; + // doc.put(varName, value); + // } else if (param instanceof Double) { + // double value = ((Double) param).doubleValue(); + // doc.put(varName, value); + // } else if (param instanceof Float) { + // float value = ((Float) param).floatValue(); + // doc.put(varName, value); + // } else if (param instanceof Long) { + // long value = ((Long) param).longValue(); + // doc.put(varName, value); + // } else if (param instanceof Boolean) { + // boolean value = ((Boolean) param).booleanValue(); + // doc.put(varName, value); + // } + // field.setAccessible(accessFlag); + // } + // return doc; + // } - public static T doc2Obj(Document doc, Class clazz) throws Exception { - T obj = clazz.newInstance(); - for (String key : doc.keySet()) { - Field field = clazz.getDeclaredField(key); - field.setAccessible(true); - field.set(obj, doc.get(key)); - } - return obj; + // public static T doc2Obj(Document doc, Class clazz) throws Exception { + // T obj = clazz.newInstance(); + // for (String key : doc.keySet()) { + // Field field = clazz.getDeclaredField(key); + // field.setAccessible(true); + // field.set(obj, doc.get(key)); + // } + // return obj; } } From 1ea40571d913a1b48449983d9e73c5c2dbd12173 Mon Sep 17 00:00:00 2001 From: huangsimin <474420502@qq.com> Date: Wed, 27 Jul 2022 16:25:10 +0800 Subject: [PATCH 29/35] =?UTF-8?q?=E5=8A=A0=E5=85=A5vscode=E8=B0=83?= =?UTF-8?q?=E8=AF=95=E6=96=B9=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .vscode/launch.json | 18 ++++ .vscode/settings.json | 3 + .vscode/tasks.json | 27 ++++++ restart.sh | 2 +- .../dataflow/controller/PacketsProcessor.java | 51 ++++++------ .../dataflow/statemachine/StateMachine.java | 82 ++++++++----------- .../statemachine/StateServerFactory.java | 9 +- .../statemachine/operate/Operate.java | 2 + .../statemachine/rpc/OperateProcessor.java | 2 +- .../com/yuandian/dataflow/utils/Utils.java | 8 ++ start.sh | 4 +- stop.sh | 1 + 12 files changed, 132 insertions(+), 77 deletions(-) create mode 100644 .vscode/launch.json create mode 100644 .vscode/settings.json create mode 100644 .vscode/tasks.json create mode 100644 src/main/java/com/yuandian/dataflow/utils/Utils.java diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..db80756 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,18 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "type": "java", + "name": "Launch Server", + "request": "launch", + "mainClass": "com.yuandian.dataflow.Server", + "projectName": "dataflow", + "args": "2", + "preLaunchTask": "restart", + "postDebugTask": "stopall", + } + ] +} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..e0f15db --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "java.configuration.updateBuildConfiguration": "automatic" +} \ No newline at end of file diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 0000000..7566731 --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,27 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "restart", + "type": "shell", + "command": "sh restart.sh", + "presentation": { + "echo": true, + "reveal": "always", + "focus": false, + "panel": "shared", + "showReuseMessage": true, + "clear": false, + "close": true + } + }, + { + "label": "stopall", + "type": "shell", + "command": "sh stop.sh", + "presentation": { + "close": true + } + } + ] +} \ No newline at end of file diff --git a/restart.sh b/restart.sh index 3bb4f6e..a51605a 100755 --- a/restart.sh +++ b/restart.sh @@ -1,2 +1,2 @@ #! /bin/bash -sh stop.sh & rm raftdata/ -rf && mvn -T4 package && truncate -s 0 screenlog.0 && sh start.sh +sh stop.sh & rm raftdata/ -rf && mvn package && truncate -s 0 screenlog.0 && sh start.sh diff --git a/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java b/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java index 14edd27..2c40164 100644 --- a/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java +++ b/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java @@ -52,40 +52,45 @@ public class PacketsProcessor implements RpcProcessor(); + resp.setSuccess(true); + rpcCtx.sendResponse(resp); var ss = StateServerFactory.getStateServer(); - log.info("{} handler request.packets.size(): {}", StateServerFactory.getServerId(), request.packets.size()); + log.debug("{} handler request.packets.size(): {}", StateServerFactory.getServerId(), request.packets.size()); + + ss.readIndexState( new GenericClosure() { @Override public void run(Status status) { - - var state = this.getValue(); - var ws = state.getWorkers().get(StateServerFactory.getServerId()); - ws.setTaskQueueSize( ws.getTaskQueueSize() - request.packets.size()); - ws.setUpdateAt(Instant.now()); - - log.debug("workerState taskQueueSize: {} psize: {} state {}", ws.getTaskQueueSize(), request.packets.size(), state.getWorkers().size()); - - Operate.CallOperate(new Operate(OperateType.PUT,ws), new GenericClosure() { - @Override - public void run(Status status) { - var resp = this.getResponse(); - resp.setMsg(rpcCtx.getRemoteAddress()); - if(status.isOk()) { - resp.setSuccess(true); - log.info("{}", resp); - } else { - resp.setSuccess(false); + log.debug("status {}", status); + if(status.isOk()) { + var state = this.getValue(); + var ws = state.getWorkers().get(StateServerFactory.getServerId()); + + + ws.setTaskQueueSize( ws.getTaskQueueSize() - request.packets.size()); + ws.setUpdateAt(Instant.now()); + + log.debug("workerState taskQueueSize: {} psize: {} state {}", ws.getTaskQueueSize(), request.packets.size(), state.getWorkers().size()); + + Operate.CallOperate(new Operate(OperateType.PUT,ws), new GenericClosure() { + @Override + public void run(Status status) { + if(status.isOk()) { + log.info("{}", resp); + } } - rpcCtx.sendResponse(resp); - } - }); + }); + } + } } ); - + } @Override diff --git a/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java b/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java index cf3b6c1..28fef8e 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java @@ -20,7 +20,7 @@ import com.yuandian.dataflow.statemachine.closure.GenericClosure; import com.yuandian.dataflow.statemachine.operate.Operate; import com.yuandian.dataflow.statemachine.operate.Operate.OperateType; import com.yuandian.dataflow.statemachine.rpc.RaftResponse; - + import com.yuandian.dataflow.statemachine.rpc.OperateProcessor.OperateRequest; import com.yuandian.dataflow.statemachine.state.State; import com.yuandian.dataflow.statemachine.state.WorkerState; @@ -38,7 +38,8 @@ import lombok.extern.slf4j.Slf4j; @Slf4j public class StateMachine extends StateMachineAdapter { - // private static final Logger LOG = LoggerFactory.getLogger(StateMachine.class); + // private static final Logger LOG = + // LoggerFactory.getLogger(StateMachine.class); /** * State value 全局使用的唯一状态 @@ -70,46 +71,46 @@ public class StateMachine extends StateMachineAdapter { if (iter.done() != null) { // This task is applied by this node, get value from closure to avoid additional // parsing. - closure = (GenericClosure)iter.done(); // 只支持单一个State. 全状态机只支持一种提交 + closure = (GenericClosure) iter.done(); // 只支持单一个State. 全状态机只支持一种提交 op = closure.getValue(); - + } else { - // Have to parse FetchAddRequest from this user log. + // Have to parse FetchAddRequest from this user log. final ByteBuffer data = iter.getData(); try { - - op = SerializerManager.getSerializer(SerializerManager.Hessian2).deserialize(data.array(),Operate.class.getName()); + + op = SerializerManager.getSerializer(SerializerManager.Hessian2).deserialize( + data.array(), + Operate.class.getName()); } catch (CodecException e) { - log.info("{}", e.toString()); + log.info("{}", e.toString()); } - + } - if(op != null) { - switch(op.getType()) { + if (op != null) { + switch (op.getType()) { case PUT: - WorkerState ws = op.getValue(); + WorkerState ws = op.getValue(); state.getWorkers().put(ws.peerId, ws); - if(closure != null) { + if (closure != null) { closure.success(op); closure.run(Status.OK()); } break; case REMOVE: - if(closure != null) { + if (closure != null) { closure.success(op); closure.run(Status.OK()); } break; default: break; - + } } else { } - - iter.next(); } @@ -136,21 +137,21 @@ public class StateMachine extends StateMachineAdapter { this.leaderTerm.set(term); // 判断是否Master线程还在跑, 如果存在则中断 - if(MasterFactory.getMasterExecute().isAlive()) { + if (MasterFactory.getMasterExecute().isAlive()) { MasterFactory.getMasterExecute().interrupt(); } - - var ss = StateServerFactory.getStateServer(); - ss.readIndexState( new GenericClosure() { + + var ss = StateServerFactory.getStateServer(); + ss.readIndexState(new GenericClosure() { @Override public void run(Status status) { - - var ws = state.getWorkers().get( StateServerFactory.getServerId() ); - if(ws == null) { + + var ws = state.getWorkers().get(StateServerFactory.getServerId()); + if (ws == null) { ws = new WorkerState(StateServerFactory.getServerId()); } - + Operate op = new Operate(OperateType.PUT, ws); ss.applyOperate(op, new GenericClosure() { @Override @@ -159,7 +160,7 @@ public class StateMachine extends StateMachineAdapter { } }); } - + }); // 当成为master时候 必须启动 @@ -175,21 +176,17 @@ public class StateMachine extends StateMachineAdapter { super.onLeaderStop(status); // 判断是否Master线程还在跑, 如果存在则中断 - if(MasterFactory.getMasterExecute().isAlive()) { + if (MasterFactory.getMasterExecute().isAlive()) { MasterFactory.getMasterExecute().interrupt(); } - } - - @Override public void onShutdown() { log.debug("onShutdown"); super.onShutdown(); } - @Override public void onStartFollowing(LeaderChangeContext ctx) { @@ -197,12 +194,12 @@ public class StateMachine extends StateMachineAdapter { try { // 判断是否Master线程还在跑, 如果存在则中断 - if(MasterFactory.getMasterExecute().isAlive()) { + if (MasterFactory.getMasterExecute().isAlive()) { MasterFactory.getMasterExecute().interrupt(); } - + var ws = new WorkerState(StateServerFactory.getServerId()); - log.debug("my: {} leader id {}",StateServerFactory.getServerId(), StateServerFactory.getLeaderId()); + log.debug("my: {} leader id {}", StateServerFactory.getServerId(), StateServerFactory.getLeaderId()); var op = new Operate(OperateType.PUT, ws); @@ -212,10 +209,10 @@ public class StateMachine extends StateMachineAdapter { log.info("{} {}", status, this.getResponse()); } }); - + return; } catch (Exception e) { - log.info("{}", e.toString()); + log.info("{}", e.toString()); } super.onStartFollowing(ctx); @@ -228,12 +225,12 @@ public class StateMachine extends StateMachineAdapter { @Override public void onStopFollowing(LeaderChangeContext ctx) { - log.debug("{} {}",ctx, StateServerFactory.getCluster().getServerId()); + log.debug("{} {}", ctx, StateServerFactory.getCluster().getServerId()); var ss = StateServerFactory.getStateServer(); var ws = new WorkerState(StateServerFactory.getServerId()); - - log.debug("my: {} leader id {}",StateServerFactory.getServerId(), StateServerFactory.getLeaderId()); + + log.debug("my: {} leader id {}", StateServerFactory.getServerId(), StateServerFactory.getLeaderId()); var op = new Operate(OperateType.PUT, ws); Operate.CallOperate(op, new GenericClosure() { @@ -243,14 +240,7 @@ public class StateMachine extends StateMachineAdapter { } }); - super.onStopFollowing(ctx); } - - - - - - } diff --git a/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java b/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java index c5d724a..7521b7e 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java @@ -188,17 +188,18 @@ public class StateServerFactory { getNode().readIndex(BytesUtil.EMPTY_BYTES, new ReadIndexClosure() { @Override public void run(Status status, long index, byte[] reqCtx) { - if( status.isOk()) { + if(status.isOk()) { + // 回调失败 closure.success(ss.fsm.getState()); - closure.run(status); - } + } + closure.run(status); } } ); } - public void applyOperate(Operate op, GenericClosure closure) { + public void applyOperate(Operate op, GenericClosure closure) { // 所有的提交都必须再leader进行 if (!ss.isLeader()) { ss.handlerNotLeaderError(closure); diff --git a/src/main/java/com/yuandian/dataflow/statemachine/operate/Operate.java b/src/main/java/com/yuandian/dataflow/statemachine/operate/Operate.java index 0194b1a..df2cf67 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/operate/Operate.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/operate/Operate.java @@ -2,6 +2,7 @@ package com.yuandian.dataflow.statemachine.operate; import java.io.Serializable; +import com.alipay.sofa.jraft.Status; import com.alipay.sofa.jraft.error.RemotingException; import com.alipay.sofa.jraft.rpc.InvokeCallback; import com.yuandian.dataflow.statemachine.StateServerFactory; @@ -75,6 +76,7 @@ public class Operate implements Serializable { var resp = (RaftResponse) result; closure.setResponse(resp); closure.success(resp.getValue()); + closure.run(Status.OK()); } }, 5000); diff --git a/src/main/java/com/yuandian/dataflow/statemachine/rpc/OperateProcessor.java b/src/main/java/com/yuandian/dataflow/statemachine/rpc/OperateProcessor.java index 96a825a..4ad2efd 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/rpc/OperateProcessor.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/rpc/OperateProcessor.java @@ -64,7 +64,7 @@ public class OperateProcessor implements RpcProcessor closure = new GenericClosure() { @Override public void run(Status status) { rpcCtx.sendResponse(getResponse()); diff --git a/src/main/java/com/yuandian/dataflow/utils/Utils.java b/src/main/java/com/yuandian/dataflow/utils/Utils.java new file mode 100644 index 0000000..1ba5e8f --- /dev/null +++ b/src/main/java/com/yuandian/dataflow/utils/Utils.java @@ -0,0 +1,8 @@ +package com.yuandian.dataflow.utils; + +public class Utils { + + public static void main(String[] args) { + + } +} diff --git a/start.sh b/start.sh index 2b4c76e..3746270 100755 --- a/start.sh +++ b/start.sh @@ -10,10 +10,10 @@ sleep 1 screen -dmS raft-0 -L java -jar target/dataflow-${VERSION}.jar 0 screen -dmS raft-1 -L java -jar target/dataflow-${VERSION}.jar 1 -screen -dmS raft-2 -L java -jar target/dataflow-${VERSION}.jar 2 +# screen -dmS raft-2 -L java -jar target/dataflow-${VERSION}.jar 2 sleep 1 screen -S raft-0 -X logfile flush 0 screen -S raft-1 -X logfile flush 0 -screen -S raft-2 -X logfile flush 0 \ No newline at end of file +# screen -S raft-2 -X logfile flush 0 \ No newline at end of file diff --git a/stop.sh b/stop.sh index 12f7ef1..9c0b857 100755 --- a/stop.sh +++ b/stop.sh @@ -2,3 +2,4 @@ screen -S raft-0 -X quit screen -S raft-1 -X quit screen -S raft-2 -X quit +exit 0 From 5298c168eff768177f02c0a3748fd7d6bfab8818 Mon Sep 17 00:00:00 2001 From: huangsimin <474420502@qq.com> Date: Wed, 27 Jul 2022 18:02:24 +0800 Subject: [PATCH 30/35] =?UTF-8?q?=E8=B0=83=E8=AF=95leader?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .vscode/launch.json | 3 +- .vscode/tasks.json | 3 +- restart.sh | 3 +- .../java/com/yuandian/dataflow/Server.java | 5 +++ .../dataflow/controller/PacketsProcessor.java | 1 - .../controller/TransferLeaderProcessor.java | 39 +++++++++++++++++++ .../dataflow/statemachine/StateMachine.java | 13 ++++--- .../statemachine/rpc/OperateProcessor.java | 15 ++++++- start.sh | 13 ++++--- 9 files changed, 78 insertions(+), 17 deletions(-) create mode 100644 src/main/java/com/yuandian/dataflow/controller/TransferLeaderProcessor.java diff --git a/.vscode/launch.json b/.vscode/launch.json index db80756..979edf3 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -10,9 +10,10 @@ "request": "launch", "mainClass": "com.yuandian.dataflow.Server", "projectName": "dataflow", - "args": "2", + "args": ["2"], "preLaunchTask": "restart", "postDebugTask": "stopall", + } ] } \ No newline at end of file diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 7566731..384c8de 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -5,9 +5,10 @@ "label": "restart", "type": "shell", "command": "sh restart.sh", + "isBackground": true, "presentation": { "echo": true, - "reveal": "always", + "reveal": "silent", "focus": false, "panel": "shared", "showReuseMessage": true, diff --git a/restart.sh b/restart.sh index a51605a..39698a2 100755 --- a/restart.sh +++ b/restart.sh @@ -1,2 +1,3 @@ #! /bin/bash -sh stop.sh & rm raftdata/ -rf && mvn package && truncate -s 0 screenlog.0 && sh start.sh + +sh stop.sh & rm raftdata/ -rf && mvn package && truncate -s 0 screenlog.0 && sh start.sh diff --git a/src/main/java/com/yuandian/dataflow/Server.java b/src/main/java/com/yuandian/dataflow/Server.java index 86232a4..b3556a2 100644 --- a/src/main/java/com/yuandian/dataflow/Server.java +++ b/src/main/java/com/yuandian/dataflow/Server.java @@ -22,8 +22,11 @@ public class Server { public static String peeridstr; public static String sprPort; public static Configuration conf ; + public static void main(String[] args) throws Exception { + + String[] peers = new String[]{"localhost:4440","localhost:4441","localhost:4442"}; String[] sprPeers = new String[]{"3440","3441","3442"}; @@ -39,6 +42,8 @@ public class Server { conf = JRaftUtils.getConfiguration(String.join(",", peers)); StateServerFactory.startStateServer(peeridstr, conf); + + // System.setProperty("server.port", sprPort); // ConfigurableApplicationContext app = SpringApplication.run(Server.class, args); // StateServerFactory.setAppCxt(app); diff --git a/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java b/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java index 2c40164..6afee1d 100644 --- a/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java +++ b/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java @@ -70,7 +70,6 @@ public class PacketsProcessor implements RpcProcessor { + + @Setter + @Getter + public static class LeaderRequest implements Serializable { + PeerId peer; + } + + @Override + public void handleRequest(RpcContext rpcCtx, LeaderRequest request) { + Status status = StateServerFactory.getCluster().getRaftNode().transferLeadershipTo(request.peer); + rpcCtx.sendResponse(status); + log.debug("[TransferLeader] {} change leader to {}", status, request.peer); + } + + @Override + public String interest() { + return LeaderRequest.class.getName(); + } + +} diff --git a/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java b/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java index 28fef8e..b854fdc 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java @@ -14,8 +14,11 @@ import com.alipay.sofa.jraft.core.StateMachineAdapter; import com.alipay.sofa.jraft.entity.LeaderChangeContext; import com.alipay.sofa.jraft.error.RaftException; import com.alipay.sofa.jraft.error.RemotingException; +import com.alipay.sofa.jraft.rpc.InvokeCallback; import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader; import com.alipay.sofa.jraft.storage.snapshot.SnapshotWriter; +import com.yuandian.dataflow.Server; +import com.yuandian.dataflow.controller.TransferLeaderProcessor; import com.yuandian.dataflow.statemachine.closure.GenericClosure; import com.yuandian.dataflow.statemachine.operate.Operate; import com.yuandian.dataflow.statemachine.operate.Operate.OperateType; @@ -173,13 +176,12 @@ public class StateMachine extends StateMachineAdapter { public void onLeaderStop(final Status status) { log.debug("onLeaderStop {}", StateServerFactory.getCluster().getServerId()); this.leaderTerm.set(-1); - super.onLeaderStop(status); - // 判断是否Master线程还在跑, 如果存在则中断 if (MasterFactory.getMasterExecute().isAlive()) { MasterFactory.getMasterExecute().interrupt(); } + super.onLeaderStop(status); } @Override @@ -198,18 +200,19 @@ public class StateMachine extends StateMachineAdapter { MasterFactory.getMasterExecute().interrupt(); } + var ws = new WorkerState(StateServerFactory.getServerId()); log.debug("my: {} leader id {}", StateServerFactory.getServerId(), StateServerFactory.getLeaderId()); var op = new Operate(OperateType.PUT, ws); - - Operate.CallOperate(op, new GenericClosure() { + Operate.CallOperate(op, new GenericClosure() { @Override public void run(Status status) { log.info("{} {}", status, this.getResponse()); } }); - + + return; } catch (Exception e) { log.info("{}", e.toString()); diff --git a/src/main/java/com/yuandian/dataflow/statemachine/rpc/OperateProcessor.java b/src/main/java/com/yuandian/dataflow/statemachine/rpc/OperateProcessor.java index 4ad2efd..4c67936 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/rpc/OperateProcessor.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/rpc/OperateProcessor.java @@ -67,8 +67,19 @@ public class OperateProcessor implements RpcProcessor closure = new GenericClosure() { @Override public void run(Status status) { - rpcCtx.sendResponse(getResponse()); - log.info("{}", status); + + if(status.isOk()) { + log.info("{}", status); + rpcCtx.sendResponse(getResponse()); + return; + } + + + if(status.getRaftError() == RaftError.EPERM) { + //TODO: Not leader 需要转发 + log.info("{}", status); + } + } }; diff --git a/start.sh b/start.sh index 3746270..51c666e 100755 --- a/start.sh +++ b/start.sh @@ -1,19 +1,20 @@ #! /bin/bash screen -S raft-0 -X quit screen -S raft-1 -X quit -screen -S raft-2 -X quit +# screen -S raft-2 -X quit + + +sleep 5s + VERSION=1.0.0-SNAPSHOT - -sleep 1 - screen -dmS raft-0 -L java -jar target/dataflow-${VERSION}.jar 0 screen -dmS raft-1 -L java -jar target/dataflow-${VERSION}.jar 1 # screen -dmS raft-2 -L java -jar target/dataflow-${VERSION}.jar 2 sleep 1 -screen -S raft-0 -X logfile flush 0 -screen -S raft-1 -X logfile flush 0 +screen -S raft-0 -X logfile flush 0 +screen -S raft-1 -X logfile flush 0 # screen -S raft-2 -X logfile flush 0 \ No newline at end of file From 7795362d5a1b42eb8bc73fa5a3e45cf61fc4ced2 Mon Sep 17 00:00:00 2001 From: huangsimin Date: Wed, 27 Jul 2022 23:37:40 +0800 Subject: [PATCH 31/35] =?UTF-8?q?TODO:=20=E8=A7=A3=E5=86=B3=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E4=B8=8D=E5=88=B0=E8=BE=BE=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .vscode/tasks.json | 16 +++-- pom.xml | 6 +- restart.sh | 3 +- .../dataflow/controller/PacketsProcessor.java | 64 +++++++++---------- .../java/com/yuandian/dataflow/projo/Doc.java | 5 +- .../dataflow/statemachine/MasterFactory.java | 16 +++-- .../dataflow/statemachine/StateMachine.java | 1 - .../statemachine/StateServerFactory.java | 31 ++++----- .../statemachine/operate/Operate.java | 2 +- .../statemachine/rpc/OperateProcessor.java | 3 +- .../statemachine/rpc/RaftResponse.java | 1 - .../java/com/yuandian/dataflow/AppTest.java | 1 + .../statemachine/StateMachineTest.java | 2 +- start.sh | 7 +- 14 files changed, 72 insertions(+), 86 deletions(-) diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 384c8de..029d8f2 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -5,24 +5,30 @@ "label": "restart", "type": "shell", "command": "sh restart.sh", - "isBackground": true, + "isBackground": false, "presentation": { "echo": true, - "reveal": "silent", + "reveal": "always", "focus": false, - "panel": "shared", + "panel": "new", "showReuseMessage": true, "clear": false, "close": true - } + + }, }, { "label": "stopall", "type": "shell", "command": "sh stop.sh", "presentation": { + "echo": true, + "reveal": "always", + "focus": false, + "panel": "shared", "close": true - } + }, + } ] } \ No newline at end of file diff --git a/pom.xml b/pom.xml index 480ac89..2d4aaf4 100644 --- a/pom.xml +++ b/pom.xml @@ -11,9 +11,9 @@ UTF-8 - 8 - 8 - 8 + 11 + 11 + 11 3.20.1 1.7.4 diff --git a/restart.sh b/restart.sh index 39698a2..8e0f02d 100755 --- a/restart.sh +++ b/restart.sh @@ -1,3 +1,4 @@ #! /bin/bash -sh stop.sh & rm raftdata/ -rf && mvn package && truncate -s 0 screenlog.0 && sh start.sh +sh stop.sh & rm raftdata/ -rf && mvn package && truncate -s 0 screenlog.0 +sh start.sh diff --git a/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java b/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java index 6afee1d..6c1b3fb 100644 --- a/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java +++ b/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java @@ -23,7 +23,7 @@ import com.yuandian.dataflow.statemachine.operate.Operate; import com.yuandian.dataflow.statemachine.operate.Operate.OperateType; import com.yuandian.dataflow.statemachine.rpc.RaftResponse; import com.yuandian.dataflow.statemachine.rpc.OperateProcessor; - + import com.yuandian.dataflow.statemachine.rpc.annotations.ProcessorRaft; import com.yuandian.dataflow.statemachine.state.State; @@ -36,7 +36,7 @@ import lombok.extern.slf4j.Slf4j; * description * * @author eson - *2022年7月21日-13:48:01 + * 2022年7月21日-13:48:01 */ @Slf4j @@ -51,45 +51,39 @@ public class PacketsProcessor implements RpcProcessor(); + resp.setSuccess(true); + rpcCtx.sendResponse(resp); - var resp = new RaftResponse<>(); - resp.setSuccess(true); - rpcCtx.sendResponse(resp); - - var ss = StateServerFactory.getStateServer(); - log.debug("{} handler request.packets.size(): {}", StateServerFactory.getServerId(), request.packets.size()); + var ss = StateServerFactory.getStateServer(); + log.debug("{} handler request.packets.size(): {}", StateServerFactory.getServerId(), request.packets.size()); - - ss.readIndexState( new GenericClosure() { + ss.readIndexState(new GenericClosure() { - @Override - public void run(Status status) { - log.debug("status {}", status); - if(status.isOk()) { - var state = this.getValue(); - var ws = state.getWorkers().get(StateServerFactory.getServerId()); + @Override + public void run(Status status) { + log.debug("status {}", status); + if (status.isOk()) { + var state = this.getValue(); + var ws = state.getWorkers().get(StateServerFactory.getServerId()); - ws.setTaskQueueSize( ws.getTaskQueueSize() - request.packets.size()); - ws.setUpdateAt(Instant.now()); - - log.debug("workerState taskQueueSize: {} psize: {} state {}", ws.getTaskQueueSize(), request.packets.size(), state.getWorkers().size()); - - Operate.CallOperate(new Operate(OperateType.PUT,ws), new GenericClosure() { - @Override - public void run(Status status) { - if(status.isOk()) { - log.info("{}", resp); - } + ws.setTaskQueueSize(ws.getTaskQueueSize() - request.packets.size()); + ws.setUpdateAt(Instant.now()); + + log.debug("workerState taskQueueSize: {} psize: {} state {}", ws.getTaskQueueSize(), + request.packets.size(), state.getWorkers().size()); + Operate.CallOperate(new Operate(OperateType.PUT, ws), new GenericClosure() { + @Override + public void run(Status status) { + if (status.isOk()) { + log.info("{}", resp); } - }); - } - + } + }); } - - } ); - - + } + }); } @Override diff --git a/src/main/java/com/yuandian/dataflow/projo/Doc.java b/src/main/java/com/yuandian/dataflow/projo/Doc.java index 3d073fd..6ee2faa 100644 --- a/src/main/java/com/yuandian/dataflow/projo/Doc.java +++ b/src/main/java/com/yuandian/dataflow/projo/Doc.java @@ -52,10 +52,7 @@ public final class Doc { @BsonProperty("responseIp") public int responseIp ; - - - - + public static void main(String[] args) { MongoClient mgo = MongoClients.create("mongodb://yuandian:yuandian123@192.168.1.113:27017"); CodecProvider pojoCodecProvider = PojoCodecProvider.builder().register("com.yuandian.dataflow.projo").build(); diff --git a/src/main/java/com/yuandian/dataflow/statemachine/MasterFactory.java b/src/main/java/com/yuandian/dataflow/statemachine/MasterFactory.java index 9263085..67725eb 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/MasterFactory.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/MasterFactory.java @@ -62,16 +62,16 @@ public class MasterFactory { WorkerState ws = state.getWorkers().get(peer); if (ws != null) { // - var canDealTasks = MAX_TASKS - ws.getTaskQueueSize(); - log.debug("cap :{} peer: {}", canDealTasks, peer); - if (canDealTasks <= 0) { + var canTasks = MAX_TASKS - ws.getTaskQueueSize(); + log.debug("cap :{} peer: {}", canTasks, peer); + if (canTasks <= 0) { return; } ws.setUpdateAt(Instant.now()); ws.setTaskQueueSize(MAX_TASKS); var request = new PacketsRequest(); - for (int i = 0; i < canDealTasks; i++) { + for (int i = 0; i < canTasks; i++) { var p = Any.pack(BacktrackingFlowOuterClass.BacktrackingFlow .newBuilder() .setTableId(10086) @@ -82,14 +82,16 @@ public class MasterFactory { Operate.CallOperate(new Operate(OperateType.PUT, ws), new GenericClosure() { @Override public void run(Status status) { - - log.info("{}", status); + log.info("PacketsRequest run {}", status); try { ss.getRpcClient().invokeAsync(peer.getEndpoint(), request, new InvokeCallback() { @Override public void complete(Object result, Throwable err) { - log.info("PacketsRequest: {}", result); + if(err != null) { + log.debug("{}", err); + } + log.debug("PacketsRequest: {}", result); } }, 5000); } catch (InterruptedException | RemotingException e) { diff --git a/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java b/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java index b854fdc..0270fba 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java @@ -211,7 +211,6 @@ public class StateMachine extends StateMachineAdapter { log.info("{} {}", status, this.getResponse()); } }); - return; } catch (Exception e) { diff --git a/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java b/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java index 7521b7e..caccbe2 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java @@ -7,6 +7,7 @@ package com.yuandian.dataflow.statemachine; import java.io.File; +import java.lang.reflect.InvocationTargetException; import java.nio.ByteBuffer; import java.util.Set; import java.util.concurrent.Executor; @@ -141,7 +142,7 @@ public class StateServerFactory { nodeOptions.setInitialConf(conf); File RaftDataFile = new File(String.format("./raftdata/%d", port) ); - log.info("{}",RaftDataFile.mkdirs()); + log.info("mkdirs: {}",RaftDataFile.mkdirs()); nodeOptions.setLogUri( String.format("./raftdata/%d/logs", port) ); nodeOptions.setRaftMetaUri(String.format("./raftdata/%d/rafts", port)); @@ -150,18 +151,14 @@ public class StateServerFactory { nodeOptions.setFsm(fsm); cluster = new RaftGroupService(groupId, serverId, nodeOptions); - - - + Set> scans = new Reflections("com.yuandian.dataflow").getTypesAnnotatedWith(ProcessorRaft.class); scans.forEach((pRaftClass)->{ try { - cluster.getRpcServer().registerProcessor((RpcProcessor) pRaftClass.newInstance()); - } catch (InstantiationException e) { + cluster.getRpcServer().registerProcessor((RpcProcessor) pRaftClass.getDeclaredConstructor().newInstance()); + } catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | NoSuchMethodException | SecurityException e) { log.info("{}", e.toString()); - } catch (IllegalAccessException e) { - log.info("{}", e.toString()); - } + } }); node = cluster.start(); @@ -219,11 +216,9 @@ public class StateServerFactory { closure.run(new Status(RaftError.EINTERNAL, errorMsg)); } } - - - - public RaftResponse redirect() { - final RaftResponse response = new RaftResponse(); + + public RaftResponse redirect() { + final RaftResponse response = new RaftResponse(); response.setSuccess(false); if (this.node != null) { final PeerId leader = this.node.getLeaderId(); @@ -234,16 +229,12 @@ public class StateServerFactory { return response; } - public void handlerNotLeaderError(final GenericClosure closure) { + public void handlerNotLeaderError(final GenericClosure closure) { closure.failure("Not leader.", redirect().getRedirect()); closure.run(new Status(RaftError.EPERM, "Not leader")); } - - + private Executor createReadIndexExecutor() { - // final StoreEngineOptions opts = new StoreEngineOptions(); - // return StoreEngineHelper.createReadIndexExecutor(opts.getReadIndexCoreThreads()); - return ThreadPoolUtil.newBuilder() // .poolName("ReadIndexPool") // .enableMetric(true) // diff --git a/src/main/java/com/yuandian/dataflow/statemachine/operate/Operate.java b/src/main/java/com/yuandian/dataflow/statemachine/operate/Operate.java index df2cf67..3b822af 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/operate/Operate.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/operate/Operate.java @@ -71,7 +71,7 @@ public class Operate implements Serializable { @Override public void complete(Object result, Throwable err) { - log.info("{}", result); + log.info("Object result {}", result); //TODO: 解决回调的次序问题 var resp = (RaftResponse) result; closure.setResponse(resp); diff --git a/src/main/java/com/yuandian/dataflow/statemachine/rpc/OperateProcessor.java b/src/main/java/com/yuandian/dataflow/statemachine/rpc/OperateProcessor.java index 4c67936..86aae22 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/rpc/OperateProcessor.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/rpc/OperateProcessor.java @@ -73,8 +73,7 @@ public class OperateProcessor implements RpcProcessor implements Serializable { private T value; - private boolean success; /** * redirect peer id diff --git a/src/test/java/com/yuandian/dataflow/AppTest.java b/src/test/java/com/yuandian/dataflow/AppTest.java index 88cdb2f..e5ba148 100644 --- a/src/test/java/com/yuandian/dataflow/AppTest.java +++ b/src/test/java/com/yuandian/dataflow/AppTest.java @@ -40,6 +40,7 @@ import lombok.extern.slf4j.Slf4j; */ @DisplayName("AppTest") @Slf4j +@var public class AppTest { diff --git a/src/test/java/com/yuandian/dataflow/statemachine/StateMachineTest.java b/src/test/java/com/yuandian/dataflow/statemachine/StateMachineTest.java index 5e117a0..4bb07f5 100644 --- a/src/test/java/com/yuandian/dataflow/statemachine/StateMachineTest.java +++ b/src/test/java/com/yuandian/dataflow/statemachine/StateMachineTest.java @@ -13,7 +13,7 @@ import com.yuandian.dataflow.statemachine.rpc.RaftResponse; import com.yuandian.dataflow.statemachine.rpc.OperateProcessor.OperateRequest; import com.yuandian.dataflow.statemachine.state.State; -import lombok.var; + import lombok.extern.slf4j.Slf4j; @Slf4j diff --git a/start.sh b/start.sh index 51c666e..0ffea46 100755 --- a/start.sh +++ b/start.sh @@ -3,17 +3,14 @@ screen -S raft-0 -X quit screen -S raft-1 -X quit # screen -S raft-2 -X quit - - - -sleep 5s +sleep 1s VERSION=1.0.0-SNAPSHOT screen -dmS raft-0 -L java -jar target/dataflow-${VERSION}.jar 0 screen -dmS raft-1 -L java -jar target/dataflow-${VERSION}.jar 1 # screen -dmS raft-2 -L java -jar target/dataflow-${VERSION}.jar 2 -sleep 1 +sleep 0.5s screen -S raft-0 -X logfile flush 0 screen -S raft-1 -X logfile flush 0 From 9a9b28799d5ff42134648a3a3bb8c123cd6cec4c Mon Sep 17 00:00:00 2001 From: huangsimin <474420502@qq.com> Date: Thu, 28 Jul 2022 14:49:56 +0800 Subject: [PATCH 32/35] =?UTF-8?q?=E5=88=9D=E7=89=88=E4=BE=8B=E5=AD=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/yuandian/dataflow/Server.java | 15 +- .../dataflow/controller/PacketsProcessor.java | 29 +- .../controller/TransferLeaderProcessor.java | 8 +- .../dataflow/grpc/CollectPackets.java | 2 +- .../dataflow/statemachine/MasterFactory.java | 116 ++++---- .../dataflow/statemachine/StateFactory.java | 272 ++++++++++++++++++ .../dataflow/statemachine/StateMachine.java | 93 +++--- .../statemachine/StateServerFactory.java | 259 ----------------- .../statemachine/closure/GenericClosure.java | 13 +- .../statemachine/closure/StateClosure.java | 5 - .../statemachine/operate/Operate.java | 22 +- .../statemachine/rpc/OperateProcessor.java | 8 +- .../statemachine/state/WorkerState.java | 12 +- src/main/resources/logback.xml | 2 +- .../java/com/yuandian/dataflow/AppTest.java | 4 +- 15 files changed, 457 insertions(+), 403 deletions(-) create mode 100644 src/main/java/com/yuandian/dataflow/statemachine/StateFactory.java delete mode 100644 src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java delete mode 100644 src/main/java/com/yuandian/dataflow/statemachine/closure/StateClosure.java diff --git a/src/main/java/com/yuandian/dataflow/Server.java b/src/main/java/com/yuandian/dataflow/Server.java index b3556a2..7b3723e 100644 --- a/src/main/java/com/yuandian/dataflow/Server.java +++ b/src/main/java/com/yuandian/dataflow/Server.java @@ -2,11 +2,13 @@ package com.yuandian.dataflow; +import org.slf4j.MarkerFactory; + import com.alipay.sofa.jraft.JRaftUtils; import com.alipay.sofa.jraft.conf.Configuration; -import com.yuandian.dataflow.statemachine.StateServerFactory; +import com.yuandian.dataflow.statemachine.StateFactory; -import lombok.var; + import lombok.extern.slf4j.Slf4j; @@ -25,22 +27,19 @@ public class Server { public static void main(String[] args) throws Exception { - - String[] peers = new String[]{"localhost:4440","localhost:4441","localhost:4442"}; - String[] sprPeers = new String[]{"3440","3441","3442"}; - + var peeridx = Integer.parseInt(args[0]); var peeridstr = peers[ peeridx ]; - var sprPort = sprPeers[ peeridx ]; + // var peeridstr = peers[2]; // var sprPort = sprPeers[2]; log.info("{} {}", peeridstr, sprPort); conf = JRaftUtils.getConfiguration(String.join(",", peers)); - StateServerFactory.startStateServer(peeridstr, conf); + StateFactory.startStateServer(peeridstr, conf); diff --git a/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java b/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java index 6c1b3fb..69c6117 100644 --- a/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java +++ b/src/main/java/com/yuandian/dataflow/controller/PacketsProcessor.java @@ -10,6 +10,9 @@ import java.io.Serializable; import java.time.Instant; import java.util.ArrayList; +import org.slf4j.Marker; +import org.slf4j.MarkerFactory; + import com.alipay.sofa.jraft.Status; import com.alipay.sofa.jraft.entity.PeerId; import com.alipay.sofa.jraft.error.RemotingException; @@ -17,7 +20,7 @@ import com.alipay.sofa.jraft.rpc.InvokeCallback; import com.alipay.sofa.jraft.rpc.RpcContext; import com.alipay.sofa.jraft.rpc.RpcProcessor; import com.google.protobuf.Any; -import com.yuandian.dataflow.statemachine.StateServerFactory; +import com.yuandian.dataflow.statemachine.StateFactory; import com.yuandian.dataflow.statemachine.closure.GenericClosure; import com.yuandian.dataflow.statemachine.operate.Operate; import com.yuandian.dataflow.statemachine.operate.Operate.OperateType; @@ -29,7 +32,7 @@ import com.yuandian.dataflow.statemachine.state.State; import lombok.Getter; import lombok.Setter; -import lombok.var; + import lombok.extern.slf4j.Slf4j; /** @@ -56,28 +59,28 @@ public class PacketsProcessor implements RpcProcessor 里的 getValue为 State的状态 ss.readIndexState(new GenericClosure() { @Override public void run(Status status) { - log.debug("status {}", status); + if (status.isOk()) { - var state = this.getValue(); - var ws = state.getWorkers().get(StateServerFactory.getServerId()); - + var state = this.getValue(); // 获取返回的状态 + var ws = state.getWorkers().get(StateFactory.getServerId()); ws.setTaskQueueSize(ws.getTaskQueueSize() - request.packets.size()); - ws.setUpdateAt(Instant.now()); + ws.setUpdateAt(Instant.now()); // 设置更新时间 - log.debug("workerState taskQueueSize: {} psize: {} state {}", ws.getTaskQueueSize(), - request.packets.size(), state.getWorkers().size()); - Operate.CallOperate(new Operate(OperateType.PUT, ws), new GenericClosure() { + // log.debug("workerState taskQueueSize: {} psize: {} state {}", ws.getTaskQueueSize(), + // request.packets.size(), state.getWorkers().size()); + Operate.CallOperate(new Operate(OperateType.PUT_WORKERSTATE, ws), new GenericClosure() { @Override public void run(Status status) { if (status.isOk()) { - log.info("{}", resp); + log.info("[{}] {}", StateFactory.getServerId(), resp); } } }); diff --git a/src/main/java/com/yuandian/dataflow/controller/TransferLeaderProcessor.java b/src/main/java/com/yuandian/dataflow/controller/TransferLeaderProcessor.java index 8372ff6..103133b 100644 --- a/src/main/java/com/yuandian/dataflow/controller/TransferLeaderProcessor.java +++ b/src/main/java/com/yuandian/dataflow/controller/TransferLeaderProcessor.java @@ -7,13 +7,17 @@ import com.alipay.sofa.jraft.entity.PeerId; import com.alipay.sofa.jraft.rpc.RpcContext; import com.alipay.sofa.jraft.rpc.RpcProcessor; import com.yuandian.dataflow.controller.PacketsProcessor.PacketsRequest; -import com.yuandian.dataflow.statemachine.StateServerFactory; +import com.yuandian.dataflow.statemachine.StateFactory; import com.yuandian.dataflow.statemachine.rpc.annotations.ProcessorRaft; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; + +/** + * 例子 强制转换leader + */ @Slf4j @ProcessorRaft public class TransferLeaderProcessor implements RpcProcessor { @@ -26,7 +30,7 @@ public class TransferLeaderProcessor implements RpcProcessor() { + + // 读一致性 + StateFactory.readIndexState(new GenericClosure() { @Override public void run(Status status) { var state = this.getValue(); - + // log.debug("masterExecute start {} {}", status, alivePeers); alivePeers.forEach((peer) -> { - WorkerState ws = state.getWorkers().get(peer); - if (ws != null) { - // - var canTasks = MAX_TASKS - ws.getTaskQueueSize(); - log.debug("cap :{} peer: {}", canTasks, peer); - if (canTasks <= 0) { - return; - } - ws.setUpdateAt(Instant.now()); - ws.setTaskQueueSize(MAX_TASKS); - - var request = new PacketsRequest(); - for (int i = 0; i < canTasks; i++) { - var p = Any.pack(BacktrackingFlowOuterClass.BacktrackingFlow - .newBuilder() - .setTableId(10086) - .build()); - request.getPackets().add(p); - } - - Operate.CallOperate(new Operate(OperateType.PUT, ws), new GenericClosure() { - @Override - public void run(Status status) { - log.info("PacketsRequest run {}", status); - try { - ss.getRpcClient().invokeAsync(peer.getEndpoint(), request, - new InvokeCallback() { - @Override - public void complete(Object result, Throwable err) { - if(err != null) { - log.debug("{}", err); - } - log.debug("PacketsRequest: {}", result); - } - }, 5000); - } catch (InterruptedException | RemotingException e) { - log.info("error send packets {}", e.toString()); - } - } - }); + + if (state == null) { + log.error("readIndexState获取的状态为 {}", state); + return; } + + WorkerState ws = state.getWorkers().get(peer); + if (ws == null) { + log.error("WorkerState获取的状态为 {}", ws); + return; + } + + var canTasks = MAX_TASKS - ws.getTaskQueueSize(); + log.info("剩余能处理的任务数量[{}] :{}", peer, canTasks); + if (canTasks <= 0) { + return; + } + ws.setUpdateAt(Instant.now()); + ws.setTaskQueueSize(MAX_TASKS); + + // 模拟发送包的数据到该节点上 + var request = new PacketsRequest(); + for (int i = 0; i < canTasks; i++) { + var p = Any.pack(BacktrackingFlowOuterClass.BacktrackingFlow + .newBuilder() + .setTableId(10086) + .build()); + request.getPackets().add(p); + } + + + // 先提交 节点的 剩余能处理的任务数量. 然后再处理 + Operate.CallOperate(new Operate(OperateType.PUT_WORKERSTATE, ws), + new GenericClosure() { + @Override + public void run(Status status) { + log.info("PacketsRequest run {}", status); + try { + StateFactory.rpcClientInvokeAsync(peer.getEndpoint(), request, + new InvokeCallback() { + @Override + public void complete(Object result, Throwable err) { + if (err != null) { + // TODO: 如果错误, 需要让节点恢复任务处理的状态 + log.debug("{}", err); + } + log.debug("PacketsRequest: {}", result); + } + }, 5000); + } catch (InterruptedException | RemotingException e) { + log.info("error send packets {}", e.toString()); + } + } + }); + }); } - - } ); + + }); } Thread.sleep(5000); diff --git a/src/main/java/com/yuandian/dataflow/statemachine/StateFactory.java b/src/main/java/com/yuandian/dataflow/statemachine/StateFactory.java new file mode 100644 index 0000000..9399243 --- /dev/null +++ b/src/main/java/com/yuandian/dataflow/statemachine/StateFactory.java @@ -0,0 +1,272 @@ +/** + * description + * + * @author eson + *2022年7月12日-13:36:24 + */ +package com.yuandian.dataflow.statemachine; + +import java.io.File; +import java.lang.reflect.InvocationTargetException; +import java.nio.ByteBuffer; +import java.util.Set; +import java.util.concurrent.Executor; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.function.Consumer; +import java.util.function.Function; + +import org.reflections.Reflections; + +import com.alipay.remoting.NamedThreadFactory; +import com.alipay.remoting.exception.CodecException; +import com.alipay.remoting.serialization.SerializerManager; +import com.alipay.sofa.jraft.JRaftUtils; +import com.alipay.sofa.jraft.Node; +import com.alipay.sofa.jraft.RaftGroupService; +import com.alipay.sofa.jraft.Status; +import com.alipay.sofa.jraft.closure.ReadIndexClosure; +import com.alipay.sofa.jraft.conf.Configuration; +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.entity.Task; +import com.alipay.sofa.jraft.error.RaftError; +import com.alipay.sofa.jraft.error.RemotingException; +import com.alipay.sofa.jraft.option.CliOptions; +import com.alipay.sofa.jraft.option.NodeOptions; +import com.alipay.sofa.jraft.rpc.InvokeCallback; +import com.alipay.sofa.jraft.rpc.InvokeContext; +import com.alipay.sofa.jraft.rpc.RpcClient; +import com.alipay.sofa.jraft.rpc.RpcProcessor; +import com.alipay.sofa.jraft.rpc.impl.BoltRaftRpcFactory; +import com.alipay.sofa.jraft.util.BytesUtil; +import com.alipay.sofa.jraft.util.Endpoint; +import com.alipay.sofa.jraft.util.ThreadPoolUtil; +import com.yuandian.dataflow.statemachine.closure.GenericClosure; +import com.yuandian.dataflow.statemachine.operate.Operate; +import com.yuandian.dataflow.statemachine.rpc.RaftResponse; + +import com.yuandian.dataflow.statemachine.rpc.OperateProcessor.OperateRequest; +import com.yuandian.dataflow.statemachine.rpc.annotations.ProcessorRaft; +import com.yuandian.dataflow.statemachine.state.State; +import com.yuandian.dataflow.statemachine.state.WorkerState; + +import lombok.Getter; +import lombok.Setter; + +import lombok.extern.slf4j.Slf4j; + +/** + * description + * + * @author eson + * 2022年7月12日-13:36:24 + */ +@Slf4j +public class StateFactory { + + private static StateServer ss; + + public static void startStateServer(String peerstr, Configuration conf) throws Exception { + if (ss != null) { + throw new Exception("重复初始化 InitStateServer"); + } + ss = new StateFactory.StateServer(peerstr, conf); + } + + public static boolean isLeader() { + return ss.node.isLeader(); + } + + public static PeerId getLeaderId() { + return ss.node.getLeaderId(); + } + + public static PeerId getServerId() { + return ss.cluster.getServerId(); + } + + public static Node getNode() { + return ss.node; + } + + public static Node getRaftNode() { + return ss.cluster.getRaftNode(); + } + + public static RpcClient getRpcClient() { + return ss.getRpcClient(); + } + + public static RaftGroupService getCluster() { + return ss.getCluster(); + } + + // 获取状态服务的对象 + public static StateServer getStateServer() { + return ss; + } + + public static void readIndexState(GenericClosure closure) { + ss.readIndexState(closure); + } + + public static void applyOperate(Operate op, GenericClosure closure) { + ss.applyOperate(op, closure); + } + + public static void rpcClientInvokeAsync(final Endpoint endpoint,final Object request,final InvokeCallback callback,final long timeoutMs) + throws InterruptedException, RemotingException { + ss.getRpcClient().invokeAsync(endpoint, request, callback, timeoutMs); + } + + public static Object rpcClientInvokeSync(final Endpoint endpoint, final Object request, final long timeoutMs) + throws InterruptedException, RemotingException { + return ss.getRpcClient().invokeSync(endpoint, request, timeoutMs); + } + + public static Object rpcClientInvokeSync(final Endpoint endpoint, final Object request, final InvokeContext ctx, + final long timeoutMs) throws InterruptedException, RemotingException { + return ss.getRpcClient().invokeSync(endpoint, request, ctx, timeoutMs); + } + + @Getter + @Setter + public static class StateServer { + + RpcClient rpcClient; + + private Node node; + private RaftGroupService cluster; + private StateMachine fsm; + + private String groupId = "dataflow"; + private Executor readIndexExecutor = createReadIndexExecutor(); + + public StateServer(String addr, Configuration conf) { + // String[] peers = new + // String[]{"localhost:4440","localhost:4441","localhost:4442"}; + // String[] sprPeers = new String[]{"3440","3441","3442"}; + + // var peeridstr = peers[Integer.parseInt(serverId)]; + // var sprPort = sprPeers[Integer.parseInt(args[0])]; + + // String groupId = "jraft"; + + // conf = + // JRaftUtils.getConfiguration("localhost:4440,localhost:4441,localhost:4442"); + + PeerId serverId = JRaftUtils.getPeerId(addr); + int port = serverId.getPort(); + + NodeOptions nodeOptions = new NodeOptions(); + + nodeOptions.setElectionTimeoutMs(1000); + nodeOptions.setSnapshotLogIndexMargin(3600); + nodeOptions.setInitialConf(conf); + + File RaftDataFile = new File(String.format("./raftdata/%d", port)); + log.info("mkdirs: {}", RaftDataFile.mkdirs()); + + nodeOptions.setLogUri(String.format("./raftdata/%d/logs", port)); + nodeOptions.setRaftMetaUri(String.format("./raftdata/%d/rafts", port)); + nodeOptions.setSnapshotUri(String.format("./raftdata/%d/snaps", port)); + fsm = new StateMachine(); // 状态实例初始化 + nodeOptions.setFsm(fsm); + + cluster = new RaftGroupService(groupId, serverId, nodeOptions); + + Set> scans = new Reflections("com.yuandian.dataflow").getTypesAnnotatedWith(ProcessorRaft.class); + scans.forEach((pRaftClass) -> { + try { + cluster.getRpcServer() + .registerProcessor((RpcProcessor) pRaftClass.getDeclaredConstructor().newInstance()); + } catch (InstantiationException | IllegalAccessException | IllegalArgumentException + | InvocationTargetException | NoSuchMethodException | SecurityException e) { + log.info("{}", e.toString()); + } + }); + node = cluster.start(); + + rpcClient = new BoltRaftRpcFactory().createRpcClient(); + rpcClient.init(new CliOptions()); + } + + public boolean isLeader() { + return this.fsm.isLeader(); + } + + public void readIndexState(GenericClosure closure) { + getNode().readIndex(BytesUtil.EMPTY_BYTES, new ReadIndexClosure() { + @Override + public void run(Status status, long index, byte[] reqCtx) { + log.debug("readIndexState({}) {}", getServerId(), status); + if (status.isOk()) { + // 回调失败 + closure.success(ss.fsm.getState()); + closure.setValue(ss.fsm.getState()); + } + closure.run(status); + } + }); + } + + public void applyOperate(Operate op, GenericClosure closure) { + // 所有的提交都必须再leader进行 + if (!ss.isLeader()) { + ss.handlerNotLeaderError(closure); + return; + } + + try { + closure.setValue(op); + final Task task = new Task(); + task.setData( + ByteBuffer.wrap(SerializerManager.getSerializer(SerializerManager.Hessian2).serialize(op))); + task.setDone(closure); // 确认所有数据 一致, 不需要加锁 + StateFactory.getStateServer().getNode().apply(task); + } catch (CodecException e) { + String errorMsg = "Fail to encode TaskState"; + log.debug(errorMsg, e); + closure.failure(errorMsg, PeerId.emptyPeer()); + closure.run(new Status(RaftError.EINTERNAL, errorMsg)); + } + } + + public RaftResponse redirect() { + final RaftResponse response = new RaftResponse(); + response.setSuccess(false); + if (this.node != null) { + final PeerId leader = this.node.getLeaderId(); + if (leader != null) { + response.setRedirect(leader); + } + } + return response; + } + + public void handlerNotLeaderError(final GenericClosure closure) { + closure.failure("Not leader.", redirect().getRedirect()); + closure.run(new Status(RaftError.EPERM, "Not leader")); + } + + private Executor createReadIndexExecutor() { + return ThreadPoolUtil.newBuilder() // + .poolName("ReadIndexPool") // + .enableMetric(true) // + .coreThreads(4) // + .maximumThreads(4) // + .keepAliveSeconds(60L) // + .workQueue(new SynchronousQueue<>()) // + .threadFactory(new NamedThreadFactory("ReadIndexService", true)) // + .rejectedHandler(new ThreadPoolExecutor.AbortPolicy()) // + .build(); + } + } + + public static void main(String[] args) throws InterruptedException, RemotingException { + var rpcClient = new BoltRaftRpcFactory().createRpcClient(); + rpcClient.init(new CliOptions()); + var resp = rpcClient.invokeSync(new Endpoint("localhost", 4441), new OperateRequest(), 5000); + log.info("{}", resp); + } +} diff --git a/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java b/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java index 0270fba..2409578 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java @@ -28,7 +28,7 @@ import com.yuandian.dataflow.statemachine.rpc.OperateProcessor.OperateRequest; import com.yuandian.dataflow.statemachine.state.State; import com.yuandian.dataflow.statemachine.state.WorkerState; -import lombok.var; + import lombok.extern.slf4j.Slf4j; /** @@ -72,13 +72,13 @@ public class StateMachine extends StateMachineAdapter { Operate op = null; GenericClosure closure = null; if (iter.done() != null) { - // This task is applied by this node, get value from closure to avoid additional - // parsing. - closure = (GenericClosure) iter.done(); // 只支持单一个State. 全状态机只支持一种提交 + + // leader可以直接从 回调closure里提取operate + closure = (GenericClosure)iter.done(); // 只支持单一个State. 全状态机只支持一种提交 op = closure.getValue(); } else { - // Have to parse FetchAddRequest from this user log. + // 非leader 需要从getData反序列化出来后处理 final ByteBuffer data = iter.getData(); try { @@ -91,29 +91,34 @@ public class StateMachine extends StateMachineAdapter { } - if (op != null) { - switch (op.getType()) { - case PUT: - WorkerState ws = op.getValue(); - state.getWorkers().put(ws.peerId, ws); - if (closure != null) { - closure.success(op); - closure.run(Status.OK()); - } - break; - case REMOVE: - if (closure != null) { - closure.success(op); - closure.run(Status.OK()); - } - break; - default: - break; + if (op == null) { + log.error("op 为 {}. 存在错误, 可能版本不一致", op); + continue; + } - } - } else { + switch (op.getType()) { + + case PUT_WORKERSTATE: + + WorkerState ws = op.getValue(); + log.debug("PUT {}", ws.peerId); + state.getWorkers().put(ws.peerId, ws); + if (closure != null) { + closure.success(op); + closure.run(Status.OK()); + } + break; + case REMOVE: + if (closure != null) { + closure.success(op); + closure.run(Status.OK()); + } + break; + default: + break; } + iter.next(); } @@ -136,7 +141,7 @@ public class StateMachine extends StateMachineAdapter { @Override public void onLeaderStart(final long term) { - log.debug("onLeaderStart {}", StateServerFactory.getServerId()); + log.debug("onLeaderStart {}", StateFactory.getServerId()); this.leaderTerm.set(term); // 判断是否Master线程还在跑, 如果存在则中断 @@ -144,26 +149,23 @@ public class StateMachine extends StateMachineAdapter { MasterFactory.getMasterExecute().interrupt(); } - var ss = StateServerFactory.getStateServer(); - ss.readIndexState(new GenericClosure() { - + + StateFactory.readIndexState(new GenericClosure() { @Override public void run(Status status) { - var ws = state.getWorkers().get(StateServerFactory.getServerId()); + var ws = state.getWorkers().get(StateFactory.getServerId()); if (ws == null) { - ws = new WorkerState(StateServerFactory.getServerId()); + ws = new WorkerState(StateFactory.getServerId()); } - Operate op = new Operate(OperateType.PUT, ws); - ss.applyOperate(op, new GenericClosure() { + StateFactory.applyOperate(new Operate(OperateType.PUT_WORKERSTATE, ws), new GenericClosure() { @Override public void run(Status status) { log.debug("master update workerstate: {}", status); } }); } - }); // 当成为master时候 必须启动 @@ -174,7 +176,7 @@ public class StateMachine extends StateMachineAdapter { @Override public void onLeaderStop(final Status status) { - log.debug("onLeaderStop {}", StateServerFactory.getCluster().getServerId()); + log.debug("onLeaderStop {}", StateFactory.getCluster().getServerId()); this.leaderTerm.set(-1); // 判断是否Master线程还在跑, 如果存在则中断 if (MasterFactory.getMasterExecute().isAlive()) { @@ -192,7 +194,7 @@ public class StateMachine extends StateMachineAdapter { @Override public void onStartFollowing(LeaderChangeContext ctx) { - log.debug("[onStartFollowing] {} {}", ctx, StateServerFactory.getCluster().getServerId()); + log.debug("[onStartFollowing] {} {}", ctx, StateFactory.getCluster().getServerId()); try { // 判断是否Master线程还在跑, 如果存在则中断 @@ -201,14 +203,13 @@ public class StateMachine extends StateMachineAdapter { } - var ws = new WorkerState(StateServerFactory.getServerId()); - log.debug("my: {} leader id {}", StateServerFactory.getServerId(), StateServerFactory.getLeaderId()); + var ws = new WorkerState(StateFactory.getServerId()); + log.debug("my: {} leader id {}", StateFactory.getServerId(), StateFactory.getLeaderId()); - var op = new Operate(OperateType.PUT, ws); - Operate.CallOperate(op, new GenericClosure() { + Operate.CallOperate(new Operate(OperateType.PUT_WORKERSTATE, ws), new GenericClosure() { @Override public void run(Status status) { - log.info("{} {}", status, this.getResponse()); + log.info("onStartFollowing CallOperate {} {}", status, this.getResponse()); } }); @@ -227,14 +228,14 @@ public class StateMachine extends StateMachineAdapter { @Override public void onStopFollowing(LeaderChangeContext ctx) { - log.debug("{} {}", ctx, StateServerFactory.getCluster().getServerId()); + log.debug("{} {}", ctx, StateFactory.getCluster().getServerId()); - var ss = StateServerFactory.getStateServer(); - var ws = new WorkerState(StateServerFactory.getServerId()); + var ss = StateFactory.getStateServer(); + var ws = new WorkerState(StateFactory.getServerId()); - log.debug("my: {} leader id {}", StateServerFactory.getServerId(), StateServerFactory.getLeaderId()); + log.debug("my: {} leader id {}", StateFactory.getServerId(), StateFactory.getLeaderId()); - var op = new Operate(OperateType.PUT, ws); + var op = new Operate(OperateType.PUT_WORKERSTATE, ws); Operate.CallOperate(op, new GenericClosure() { @Override public void run(Status status) { diff --git a/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java b/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java deleted file mode 100644 index caccbe2..0000000 --- a/src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java +++ /dev/null @@ -1,259 +0,0 @@ -/** - * description - * - * @author eson - *2022年7月12日-13:36:24 - */ -package com.yuandian.dataflow.statemachine; - -import java.io.File; -import java.lang.reflect.InvocationTargetException; -import java.nio.ByteBuffer; -import java.util.Set; -import java.util.concurrent.Executor; -import java.util.concurrent.SynchronousQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.function.Consumer; -import java.util.function.Function; - -import org.reflections.Reflections; - -import com.alipay.remoting.NamedThreadFactory; -import com.alipay.remoting.exception.CodecException; -import com.alipay.remoting.serialization.SerializerManager; -import com.alipay.sofa.jraft.JRaftUtils; -import com.alipay.sofa.jraft.Node; -import com.alipay.sofa.jraft.RaftGroupService; -import com.alipay.sofa.jraft.Status; -import com.alipay.sofa.jraft.closure.ReadIndexClosure; -import com.alipay.sofa.jraft.conf.Configuration; -import com.alipay.sofa.jraft.entity.PeerId; -import com.alipay.sofa.jraft.entity.Task; -import com.alipay.sofa.jraft.error.RaftError; -import com.alipay.sofa.jraft.error.RemotingException; -import com.alipay.sofa.jraft.option.CliOptions; -import com.alipay.sofa.jraft.option.NodeOptions; -import com.alipay.sofa.jraft.rpc.InvokeCallback; -import com.alipay.sofa.jraft.rpc.RpcClient; -import com.alipay.sofa.jraft.rpc.RpcProcessor; -import com.alipay.sofa.jraft.rpc.impl.BoltRaftRpcFactory; -import com.alipay.sofa.jraft.util.BytesUtil; -import com.alipay.sofa.jraft.util.Endpoint; -import com.alipay.sofa.jraft.util.ThreadPoolUtil; -import com.yuandian.dataflow.statemachine.closure.GenericClosure; -import com.yuandian.dataflow.statemachine.operate.Operate; -import com.yuandian.dataflow.statemachine.rpc.RaftResponse; - -import com.yuandian.dataflow.statemachine.rpc.OperateProcessor.OperateRequest; -import com.yuandian.dataflow.statemachine.rpc.annotations.ProcessorRaft; -import com.yuandian.dataflow.statemachine.state.State; -import com.yuandian.dataflow.statemachine.state.WorkerState; - -import lombok.Getter; -import lombok.Setter; -import lombok.var; -import lombok.extern.slf4j.Slf4j; - -/** - * description - * - * @author eson - *2022年7月12日-13:36:24 - */ -@Slf4j -@var -public class StateServerFactory { - - private static StateServer ss; - - public static void startStateServer(String peerstr, Configuration conf) throws Exception { - if(ss != null) { - throw new Exception("重复初始化 InitStateServer"); - } - ss = new StateServerFactory.StateServer(peerstr, conf); - } - - - public static boolean isLeader() { - return ss.node.isLeader() ; - } - - - public static PeerId getLeaderId() { - return ss.node.getLeaderId() ; - } - - public static PeerId getServerId() { - return ss.cluster.getServerId(); - } - - public static Node getNode() { - return ss.node ; - } - - public static Node getRaftNode() { - return ss.cluster.getRaftNode() ; - } - - public static RpcClient getRpcClient() { - return ss.getRpcClient(); - } - - public static RaftGroupService getCluster() { - return ss.getCluster(); - } - - // 获取状态服务的对象 - public static StateServer getStateServer() { - return ss; - } - - @Getter - @Setter - public static class StateServer { - - RpcClient rpcClient; - - private Node node; - private RaftGroupService cluster; - private StateMachine fsm; - - private String groupId = "dataflow"; - private Executor readIndexExecutor = createReadIndexExecutor(); - - public StateServer(String addr, Configuration conf) { - // String[] peers = new String[]{"localhost:4440","localhost:4441","localhost:4442"}; - // String[] sprPeers = new String[]{"3440","3441","3442"}; - - // var peeridstr = peers[Integer.parseInt(serverId)]; - // var sprPort = sprPeers[Integer.parseInt(args[0])]; - - // String groupId = "jraft"; - - // conf = JRaftUtils.getConfiguration("localhost:4440,localhost:4441,localhost:4442"); - - PeerId serverId = JRaftUtils.getPeerId(addr); - int port = serverId.getPort(); - - NodeOptions nodeOptions = new NodeOptions(); - - nodeOptions.setElectionTimeoutMs(1000); - nodeOptions.setSnapshotLogIndexMargin(3600); - nodeOptions.setInitialConf(conf); - - File RaftDataFile = new File(String.format("./raftdata/%d", port) ); - log.info("mkdirs: {}",RaftDataFile.mkdirs()); - - nodeOptions.setLogUri( String.format("./raftdata/%d/logs", port) ); - nodeOptions.setRaftMetaUri(String.format("./raftdata/%d/rafts", port)); - nodeOptions.setSnapshotUri(String.format("./raftdata/%d/snaps", port)); - fsm = new StateMachine(); // 状态实例初始化 - nodeOptions.setFsm(fsm); - - cluster = new RaftGroupService(groupId, serverId, nodeOptions); - - Set> scans = new Reflections("com.yuandian.dataflow").getTypesAnnotatedWith(ProcessorRaft.class); - scans.forEach((pRaftClass)->{ - try { - cluster.getRpcServer().registerProcessor((RpcProcessor) pRaftClass.getDeclaredConstructor().newInstance()); - } catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | NoSuchMethodException | SecurityException e) { - log.info("{}", e.toString()); - } - }); - node = cluster.start(); - - rpcClient = new BoltRaftRpcFactory().createRpcClient(); - rpcClient.init(new CliOptions()); - } - - - - public boolean isLeader() { - return this.fsm.isLeader(); - } - - - - public void useFsmState(Consumer dofunc) { - var state = ss.fsm.getState(); - synchronized(state) { - dofunc.accept(state); - } - } - - public void readIndexState(GenericClosure closure) { - getNode().readIndex(BytesUtil.EMPTY_BYTES, new ReadIndexClosure() { - @Override - public void run(Status status, long index, byte[] reqCtx) { - if(status.isOk()) { - // 回调失败 - closure.success(ss.fsm.getState()); - } - closure.run(status); - } - } ); - } - - - - public void applyOperate(Operate op, GenericClosure closure) { - // 所有的提交都必须再leader进行 - if (!ss.isLeader()) { - ss.handlerNotLeaderError(closure); - return; - } - - try { - closure.setValue(op); - final Task task = new Task(); - task.setData(ByteBuffer.wrap(SerializerManager.getSerializer(SerializerManager.Hessian2).serialize(op))); - task.setDone(closure); // 确认所有数据 一致, 不需要加锁 - StateServerFactory.getStateServer().getNode().apply(task); - } catch (CodecException e) { - String errorMsg = "Fail to encode TaskState"; - log.debug(errorMsg, e); - closure.failure(errorMsg, PeerId.emptyPeer()); - closure.run(new Status(RaftError.EINTERNAL, errorMsg)); - } - } - - public RaftResponse redirect() { - final RaftResponse response = new RaftResponse(); - response.setSuccess(false); - if (this.node != null) { - final PeerId leader = this.node.getLeaderId(); - if (leader != null) { - response.setRedirect(leader); - } - } - return response; - } - - public void handlerNotLeaderError(final GenericClosure closure) { - closure.failure("Not leader.", redirect().getRedirect()); - closure.run(new Status(RaftError.EPERM, "Not leader")); - } - - private Executor createReadIndexExecutor() { - return ThreadPoolUtil.newBuilder() // - .poolName("ReadIndexPool") // - .enableMetric(true) // - .coreThreads(4) // - .maximumThreads(4) // - .keepAliveSeconds(60L) // - .workQueue(new SynchronousQueue<>()) // - .threadFactory(new NamedThreadFactory("ReadIndexService", true)) // - .rejectedHandler(new ThreadPoolExecutor.AbortPolicy()) // - .build(); - } -} - - - - public static void main(String[] args) throws InterruptedException, RemotingException { - var rpcClient = new BoltRaftRpcFactory().createRpcClient(); - rpcClient.init(new CliOptions()); - var resp = rpcClient.invokeSync(new Endpoint("localhost",4441), new OperateRequest(), 5000); - log.info("{}", resp); - } -} diff --git a/src/main/java/com/yuandian/dataflow/statemachine/closure/GenericClosure.java b/src/main/java/com/yuandian/dataflow/statemachine/closure/GenericClosure.java index 002bed1..db6787a 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/closure/GenericClosure.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/closure/GenericClosure.java @@ -3,6 +3,7 @@ package com.yuandian.dataflow.statemachine.closure; import com.alipay.sofa.jraft.Closure; import com.alipay.sofa.jraft.Status; import com.alipay.sofa.jraft.entity.PeerId; +import com.yuandian.dataflow.statemachine.StateFactory; import com.yuandian.dataflow.statemachine.operate.Operate; import com.yuandian.dataflow.statemachine.rpc.RaftResponse; import com.yuandian.dataflow.statemachine.state.State; @@ -21,15 +22,21 @@ import org.slf4j.LoggerFactory; @ToString public abstract class GenericClosure implements Closure { + // 状态机的统一响应 private RaftResponse response; // 代表任务状态 private T value; public GenericClosure() { - + } + /** + * 错误的时候返回错误信息. 自动装配response + * @param errorMsg + * @param redirect + */ public void failure(final String errorMsg, final PeerId redirect) { final RaftResponse response = new RaftResponse(); response.setSuccess(false); @@ -38,6 +45,10 @@ public abstract class GenericClosure implements Closure { setResponse(response); } + /** + * 成功时调用该方法. 自动装配response + * @param value + */ public void success(final T value) { final RaftResponse response = new RaftResponse(); response.setValue(value); diff --git a/src/main/java/com/yuandian/dataflow/statemachine/closure/StateClosure.java b/src/main/java/com/yuandian/dataflow/statemachine/closure/StateClosure.java deleted file mode 100644 index e49a805..0000000 --- a/src/main/java/com/yuandian/dataflow/statemachine/closure/StateClosure.java +++ /dev/null @@ -1,5 +0,0 @@ -package com.yuandian.dataflow.statemachine.closure; - -public class StateClosure { - -} diff --git a/src/main/java/com/yuandian/dataflow/statemachine/operate/Operate.java b/src/main/java/com/yuandian/dataflow/statemachine/operate/Operate.java index 3b822af..d385dd2 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/operate/Operate.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/operate/Operate.java @@ -5,14 +5,14 @@ import java.io.Serializable; import com.alipay.sofa.jraft.Status; import com.alipay.sofa.jraft.error.RemotingException; import com.alipay.sofa.jraft.rpc.InvokeCallback; -import com.yuandian.dataflow.statemachine.StateServerFactory; +import com.yuandian.dataflow.statemachine.StateFactory; import com.yuandian.dataflow.statemachine.closure.GenericClosure; import com.yuandian.dataflow.statemachine.rpc.OperateProcessor; import com.yuandian.dataflow.statemachine.rpc.RaftResponse; import com.yuandian.dataflow.statemachine.state.WorkerState; import lombok.Data; -import lombok.var; + import lombok.extern.slf4j.Slf4j; /** @@ -22,11 +22,17 @@ import lombok.extern.slf4j.Slf4j; */ @Slf4j @Data -@var public class Operate implements Serializable { public static enum OperateType { - PUT, REMOVE; + /** + * 同步WorkerState状态. + */ + PUT_WORKERSTATE, + /** + * 暂无想法 + */ + REMOVE; } private OperateType type; @@ -52,10 +58,10 @@ public class Operate implements Serializable { * @param closure 回调函数. Operate为返回值 */ public static void CallOperate(Operate op, GenericClosure closure) { - - var ss = StateServerFactory.getStateServer(); + log.debug("CallOperate Value {}", op.getValue()); + var ss = StateFactory.getStateServer(); // 如果是leader 就直接提交 - if (StateServerFactory.isLeader()) { + if (StateFactory.isLeader()) { ss.applyOperate(op, closure); return; } @@ -64,7 +70,7 @@ public class Operate implements Serializable { var request = new OperateProcessor.OperateRequest(); request.setOperate(op); - var leaderId = StateServerFactory.getLeaderId(); + var leaderId = StateFactory.getLeaderId(); try { ss.getRpcClient().invokeAsync(leaderId.getEndpoint(), request, new InvokeCallback() { diff --git a/src/main/java/com/yuandian/dataflow/statemachine/rpc/OperateProcessor.java b/src/main/java/com/yuandian/dataflow/statemachine/rpc/OperateProcessor.java index 86aae22..642a111 100644 --- a/src/main/java/com/yuandian/dataflow/statemachine/rpc/OperateProcessor.java +++ b/src/main/java/com/yuandian/dataflow/statemachine/rpc/OperateProcessor.java @@ -16,8 +16,8 @@ import com.alipay.sofa.jraft.entity.Task; import com.alipay.sofa.jraft.error.RaftError; import com.alipay.sofa.jraft.rpc.RpcContext; import com.alipay.sofa.jraft.rpc.RpcProcessor; -import com.yuandian.dataflow.statemachine.StateServerFactory; -import com.yuandian.dataflow.statemachine.StateServerFactory.StateServer; +import com.yuandian.dataflow.statemachine.StateFactory; +import com.yuandian.dataflow.statemachine.StateFactory.StateServer; import com.yuandian.dataflow.statemachine.closure.GenericClosure; import com.yuandian.dataflow.statemachine.operate.Operate; import com.yuandian.dataflow.statemachine.rpc.annotations.ProcessorRaft; @@ -29,7 +29,7 @@ import org.apache.commons.lang.StringUtils; import lombok.Getter; import lombok.Setter; import lombok.ToString; -import lombok.var; + import lombok.extern.slf4j.Slf4j; /** @@ -82,7 +82,7 @@ public class OperateProcessor implements RpcProcessor - %d{yyyyMMdd HH:mm:ss.SSS} %level %thread\(%file:%line\): %msg%n + %d{yyyy-MM-dd HH:mm:ss.SSS} %red(%level) %cyan(%thread\(%file:%line\)): %msg%n