Skip to content

[Issue #869] support pipline execution #868

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 46 commits into from
Jun 15, 2025
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
46 commits
Select commit Hold shift + click to select a range
6ef3369
fix no data bug in stream storage
huasiy Nov 7, 2024
3457754
resolve merge conflict
huasiy Dec 3, 2024
444425f
add broadcast join stream invoker
huasiy Oct 10, 2024
65fcbcb
add single stage join stream operator
huasiy Oct 10, 2024
4d29327
add BroadcastJoinStreamWorker
huasiy Oct 22, 2024
35aa0b8
support broadcast join stream worker with right child
huasiy Oct 24, 2024
adb83b2
modify map between workers from two stages
huasiy Oct 25, 2024
df09a64
only one worker will pass schema to down stream
huasiy Oct 28, 2024
0947cd2
one worker write to one port of down stream worker
huasiy Oct 28, 2024
4286383
pack related code
huasiy Nov 1, 2024
831b872
succeed to transfer schema
huasiy Dec 19, 2024
c181d0d
support demo query
huasiy Dec 20, 2024
8eeb709
support stream storage for partitioned join
huasiy Dec 21, 2024
a76c43c
specify worker port
huasiy Dec 23, 2024
262a3c3
specify port to write for partition worker
huasiy Dec 23, 2024
be0aeab
fix the number of workers at present
huasiy Dec 23, 2024
0a517b1
support partitioned join q12
huasiy Dec 24, 2024
4f4a879
get partition tasks from worker coordinator
huasiy Dec 24, 2024
db35168
specify broadcast join worker ports
huasiy Dec 24, 2024
a092eba
fix bug
huasiy Dec 24, 2024
f580570
fix bug
huasiy Dec 24, 2024
d49de0c
assign down stream workers for broadcast join worker
huasiy Dec 24, 2024
2f4d537
clean code
huasiy Dec 25, 2024
f86ac27
clean code
huasiy Dec 25, 2024
7f74978
invoke broadcast chain join stream worker
huasiy Dec 25, 2024
cd511fa
shutdown ManagedChannel
huasiy Dec 27, 2024
efcac87
support stream transmission between partitioned join stream workers
huasiy Dec 28, 2024
9f45582
fix some bugs
huasiy Dec 30, 2024
bb14596
add down stream workers to broadcast join stage
huasiy Dec 30, 2024
a9bd009
right child stage should know left child stage worker number
huasiy Dec 31, 2024
9e30a48
right child stage should know left child stage worker number
huasiy Dec 31, 2024
f1609e4
support q10
huasiy Jan 8, 2025
d5e2b7e
fix partitioned join worker port index bug
huasiy Jan 8, 2025
93ad17d
support q3
huasiy Jan 10, 2025
2faffd1
add partition chain join invoker and worker
huasiy Jan 10, 2025
e847565
support q5
huasiy Jan 10, 2025
d4e5e5f
invoke all workers under stream mode
huasiy Jan 17, 2025
b00cf91
don't wait for worker's readiness
huasiy Jan 17, 2025
9d5b915
Partitioned Join Stream Worker get data from multiple workers
huasiy Jan 17, 2025
7465533
Support send async in StreamOutputStream
huasiy Feb 22, 2025
54b845b
establish http connection when build writer
huasiy Mar 2, 2025
5c6a6a9
clean code
huasiy Mar 2, 2025
2270b46
StreamOutputStream support flush async
huasiy Mar 4, 2025
b2b452b
fix some bugs
huasiy Mar 15, 2025
574ada1
resolve all comments
huasiy Apr 4, 2025
e7d67f0
resolve code conflict
huasiy Jun 2, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
right child stage should know left child stage worker number
  • Loading branch information
huasiy committed Mar 30, 2025
commit a9bd00927bca351c1cc9471741dfd602f7794bf8
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,8 @@ public class StageCoordinator
private final List<Worker<CFWorkerInfo>> workers = new ArrayList<>();
private final Map<Long, List<Integer>> workerIdToWorkerIndex = new ConcurrentHashMap<>();
private int workerIndexAssigner;
private int leftChildWorkerNum;
private int rightChildWorkerNum;
private final Object lock = new Object();

/**
Expand All @@ -82,6 +84,20 @@ public StageCoordinator(int stageId, int workerNum)
this.taskQueue = null;
this.downStreamWorkerNum = 0;
this.workerIndexAssigner = 0;
this.leftChildWorkerNum = 0;
this.rightChildWorkerNum = 0;
}

public StageCoordinator(int stageId, int workerNum, int workerIndexAssigner)
{
this.stageId = stageId;
this.isQueued = false;
this.fixedWorkerNum = workerNum;
this.taskQueue = null;
this.downStreamWorkerNum = 0;
this.workerIndexAssigner = workerIndexAssigner;
this.leftChildWorkerNum = 0;
this.rightChildWorkerNum = 0;
}

/**
Expand All @@ -100,6 +116,8 @@ public StageCoordinator(int stageId, List<Task> pendingTasks, int workerIndex)
this.taskQueue = new TaskQueue<>(pendingTasks);
this.downStreamWorkerNum = 0;
this.workerIndexAssigner = workerIndex;
this.leftChildWorkerNum = 0;
this.rightChildWorkerNum = 0;
}

/**
Expand Down Expand Up @@ -284,4 +302,34 @@ public int getFixedWorkerNum()
{
return this.fixedWorkerNum;
}

public void setLeftChildWorkerNum(int num)
{
leftChildWorkerNum = num;
}

public int getLeftChildWorkerNum()
{
return leftChildWorkerNum;
}

public boolean leftChildWorkerIsEmpty()
{
return leftChildWorkerNum == 0;
}

public void setRightChildWorkerNum(int num)
{
rightChildWorkerNum = num;
}

public int getRightChildWorkerNum()
{
return rightChildWorkerNum;
}

public boolean rightChildWorkerIsEmpty()
{
return rightChildWorkerNum == 0;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,24 @@ public void initPlanCoordinator(PlanCoordinator planCoordinator, int parentStage
joinInput.setStageId(this.joinStageId);
}
StageDependency joinStageDependency = new StageDependency(joinStageId, parentStageId, wideDependOnParent);
StageCoordinator joinStageCoordinator = new StageCoordinator(joinStageId, this.joinInputs.size());
StageCoordinator parentStageCoordinator = planCoordinator.getStageCoordinator(parentStageId);
StageCoordinator joinStageCoordinator;
if (parentStageCoordinator != null)
{
if (parentStageCoordinator.leftChildWorkerIsEmpty())
{
joinStageCoordinator = new StageCoordinator(joinStageId, this.joinInputs.size());
parentStageCoordinator.setLeftChildWorkerNum(this.joinInputs.size());
} else
{
joinStageCoordinator = new StageCoordinator(joinStageId, this.joinInputs.size(),
parentStageCoordinator.getLeftChildWorkerNum());
parentStageCoordinator.setRightChildWorkerNum(this.joinInputs.size());
}
} else
{
joinStageCoordinator = new StageCoordinator(joinStageId, this.joinInputs.size());
}
planCoordinator.addStageCoordinator(joinStageCoordinator, joinStageDependency);
if (this.joinAlgo == JoinAlgorithm.PARTITIONED || this.joinAlgo == JoinAlgorithm.PARTITIONED_CHAIN)
{
Expand Down