test/dataflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

finish

Browse Source
This commit is contained in:
huangsimin
2022-07-19 00:04:59 +08:00
parent 36fe0e6428
commit 20e50d142d
8 changed files with 61 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/main/java/com/yuandian/dataflow/Server.java
View File

@@ -7,6 +7,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.SpringBootConfiguration;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration;
import com.alipay.remoting.serialization.SerializerManager;
import com.alipay.sofa.jraft.JRaftUtils;
@@ -31,7 +32,7 @@ import lombok.extern.slf4j.Slf4j;
*
*/
@Slf4j
@SpringBootApplication
@SpringBootApplication(exclude = {MongoAutoConfiguration.class})
@SpringBootConfiguration
public class Server {

12
src/main/java/com/yuandian/dataflow/controller/TaskLog.java
View File

@@ -7,10 +7,13 @@ import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestParam;
import com.alipay.sofa.jraft.Status;
import com.alipay.sofa.jraft.entity.PeerId;
import com.alipay.sofa.jraft.error.RemotingException;
import com.yuandian.dataflow.projo.Response;
import com.yuandian.dataflow.statemachine.StateServerFactory;
import com.yuandian.dataflow.statemachine.SyncClosure;
import com.yuandian.dataflow.statemachine.state.State;
import com.yuandian.dataflow.statemachine.state.WorkerState;
import lombok.var;
import lombok.extern.slf4j.Slf4j;
@@ -20,7 +23,7 @@ import lombok.extern.slf4j.Slf4j;
public class TaskLog {
@GetMapping(path = "/test")
public ResponseEntity<Response> Processing() throws InterruptedException {
public ResponseEntity<Response> Processing() throws InterruptedException, RemotingException {
StateServerFactory.getStateServer().useFsmState((fsmState)->{
@@ -31,6 +34,13 @@ public class TaskLog {
// state.getWorker().setPeerId(StateServerFactory.getStateServer().getNode().getNodeId().getPeerId());
// state.getWorker().setTaskQueueSize(1);
StateServerFactory.getStateServer().updateFsmState((fsmState)->{
log.error(fsmState.toString() );
fsmState.getWorkers().put(new PeerId(), new WorkerState());
return fsmState;
});

86
src/main/java/com/yuandian/dataflow/grpc/MongodbTest.java
View File

@@ -1,86 +0,0 @@
package com.yuandian.dataflow.grpc;
import com.mongodb.MongoClient;
import com.mongodb.MongoCredential;
import com.mongodb.ServerAddress;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoDatabase;
import org.bson.Document;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.List;
public class MongodbTest {
public static <T> void insertMsgToMongoDB(T obj) {
try {
ServerAddress serverAddress = new ServerAddress("192.168.1.113", 27017);
List<ServerAddress> addrs = new ArrayList<>();
addrs.add(serverAddress);
MongoCredential credential = MongoCredential.createScramSha1Credential("yd-rw", "admin", "yuandian123".toCharArray());
List<MongoCredential> credentials = new ArrayList<>();
credentials.add(credential);
MongoClient mongoClient = new MongoClient(addrs, credentials);
MongoDatabase db = mongoClient.getDatabase("yd-base");
// todo 修改名字
MongoCollection<Document> collection = db.getCollection("lxy-test");
collection.insertOne(obj2Doc(obj));
System.err.println("insert success");
} catch (Exception e) {
e.printStackTrace();
}
}
public static <T> Document obj2Doc(T obj) throws Exception {
Document doc = new Document();
Field[] fields = obj.getClass().getDeclaredFields();
for (Field field : fields) {
String varName = field.getName();
boolean accessFlag = field.isAccessible();
if (!accessFlag) {
field.setAccessible(true);
}
Object param = field.get(obj);
if (param == null) {
continue;
} else if (param instanceof Integer) {
int value = ((Integer) param).intValue();
doc.put(varName, value);
} else if (param instanceof String) {
String value = (String) param;
doc.put(varName, value);
} else if (param instanceof Double) {
double value = ((Double) param).doubleValue();
doc.put(varName, value);
} else if (param instanceof Float) {
float value = ((Float) param).floatValue();
doc.put(varName, value);
} else if (param instanceof Long) {
long value = ((Long) param).longValue();
doc.put(varName, value);
} else if (param instanceof Boolean) {
boolean value = ((Boolean) param).booleanValue();
doc.put(varName, value);
}
field.setAccessible(accessFlag);
}
return doc;
}
public static <T> T doc2Obj(Document doc, Class<T> clazz) throws Exception {
T obj = clazz.newInstance();
for (String key : doc.keySet()) {
Field field = clazz.getDeclaredField(key);
field.setAccessible(true);
field.set(obj, doc.get(key));
}
return obj;
}
}

22
src/main/java/com/yuandian/dataflow/projo/Doc.java
View File

@@ -2,27 +2,20 @@ package com.yuandian.dataflow.projo;
import java.io.Serializable;
import java.lang.annotation.Documented;
import java.time.LocalDateTime;
import org.bson.Document;
import org.bson.codecs.pojo.annotations.BsonCreator;
import org.bson.codecs.pojo.annotations.BsonDiscriminator;
import org.bson.codecs.pojo.annotations.BsonProperty;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.util.JSONPObject;
import lombok.Getter;
import lombok.Setter;
@Getter
@Setter
public final class Doc extends Document {
@Getter
public final class Doc extends Document {
@JsonProperty("code")
@BsonProperty("code")
@@ -33,14 +26,11 @@ public final class Doc extends Document {
public LocalDateTime TS;
@JsonProperty("desc")
@BsonProperty("desc")
@BsonProperty("desc")
public String Desc;
@JsonProperty("data")
@BsonProperty("data")
public Document Data;
@BsonProperty("data")
public Document Data;
}

36
src/main/java/com/yuandian/dataflow/statemachine/StateMachine.java
View File

@@ -80,11 +80,32 @@ public class StateMachine extends StateMachineAdapter {
/**
* Returns current value. 读取修改都在这个函数域内进行
* @throws RemotingException
* @throws InterruptedException
*/
public void updateState(Function<State, State> dofunc) {
public void updateState(Function<State, State> dofunc) throws InterruptedException, RemotingException {
synchronized(this.state) {
var newstate = dofunc.apply(this.state);
this.state = newstate;
var ss = StateServerFactory.getStateServer();
if(!isLeader()) {
var request = new RequestState();
request.setState(newstate);
var result = ss.getRpcClient().invokeSync(ss.getNode().getLeaderId().getEndpoint(), request, 5000);
log.info("{}", result);
return;
}
// this.state = newstate;
if(newstate != null) {
var colsure = new SyncClosure<State>() {
@Override
public void run(Status status) {
}
};
colsure.setValue(newstate);
StateServerFactory.getStateServer().applyState(newstate, colsure);
}
}
}
@@ -98,19 +119,22 @@ public class StateMachine extends StateMachineAdapter {
// This task is applied by this node, get value from closure to avoid additional
// parsing.
var closure = (SyncClosure<State>)iter.done(); // 只支持单一个State. 全状态机只支持一种提交
state = closure.getValue();
log.info("SyncDataClosure(done) taskState:{} leaderTerm:{}",state, this.leaderTerm);
log.info("SyncDataClosure(done) taskState:{} leaderTerm:{} {}",this.state, this.leaderTerm, closure);
synchronized(this.state) {
this.state = closure.getValue();
}
closure.success(state);
closure.run(Status.OK());
} else {
// Have to parse FetchAddRequest from this user log.
final ByteBuffer data = iter.getData();
try {
synchronized(state) {
state = SerializerManager.getSerializer(SerializerManager.Hessian2).deserialize(
data.array(), State.class.getName());
log.info("SyncDataClosure(null) taskState:{} leaderTerm:{}", state, this.leaderTerm);
}
} catch (CodecException e) {
e.printStackTrace();

10
src/main/java/com/yuandian/dataflow/statemachine/StateServerFactory.java
View File

@@ -171,6 +171,10 @@ public class StateServerFactory {
}
public void updateFsmState(Function<State, State> dofunc) throws InterruptedException, RemotingException {
this.getFsm().updateState(dofunc);
}
public void applyState(State state, SyncClosure<State> closure) {
// 所有的提交都必须再leader进行
@@ -193,7 +197,7 @@ public class StateServerFactory {
}
}
public void applyWorkerState(WorkerState state, SyncClosure<WorkerState> closure) {
public void applyWorkerState(WorkerState state, SyncClosure<State> closure) {
if (!ss.isLeader()) {
ss.handlerNotLeaderError(closure);
return;
@@ -209,13 +213,13 @@ public class StateServerFactory {
wmap.put(state.getPeerId(), state);
try {
final Task task = new Task();
log.error("{}", fsmState);
closure.setValue(fsmState);
task.setData(ByteBuffer.wrap(SerializerManager.getSerializer(SerializerManager.Hessian2).serialize(fsmState)));
task.setDone(closure);
StateServerFactory.getStateServer().getNode().apply(task); // 提交数据
} catch (CodecException e) {
String errorMsg = "Fail to encode TaskState";
log.error(errorMsg, e);
log.error("{}:{}",errorMsg, e);
closure.failure(errorMsg, PeerId.emptyPeer());
closure.run(new Status(RaftError.EINTERNAL, errorMsg));
}

3
src/main/java/com/yuandian/dataflow/statemachine/rpc/SyncConditionProcessor.java
View File

@@ -19,6 +19,7 @@ import com.lmax.disruptor.WorkProcessor;
import com.yuandian.dataflow.statemachine.StateServerFactory;
import com.yuandian.dataflow.statemachine.SyncClosure;
import com.yuandian.dataflow.statemachine.StateServerFactory.StateServer;
import com.yuandian.dataflow.statemachine.state.State;
import com.yuandian.dataflow.statemachine.state.WorkerState;
import com.alipay.sofa.jraft.entity.PeerId;
@@ -40,7 +41,7 @@ public class SyncConditionProcessor implements RpcProcessor<RequestCondition> {
public void handleRequest(RpcContext rpcCtx, RequestCondition request) {
log.info("request: {}", request);
final SyncClosure<WorkerState> closure = new SyncClosure<WorkerState>() {
final SyncClosure<State> closure = new SyncClosure< State>() {
@Override
public void run(Status status) {
rpcCtx.sendResponse(getResponse());