Skip to content

Commit 138585c

Browse files
author
Alejandro Abdelnur
committed
YARN-1021. Yarn Scheduler Load Simulator. (ywskycn via tucu)
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1527065 13f79535-47bb-0310-9956-ffa450edef68
1 parent 157f0f2 commit 138585c

File tree

71 files changed

+36836
-0
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

71 files changed

+36836
-0
lines changed

‎hadoop-assemblies/src/main/resources/assemblies/hadoop-sls.xml‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
<!--
2+
Licensed to the Apache Software Foundation (ASF) under one
3+
or more contributor license agreements. See the NOTICE file
4+
distributed with this work for additional information
5+
regarding copyright ownership. The ASF licenses this file
6+
to you under the Apache License, Version 2.0 (the
7+
"License"); you may not use this file except in compliance
8+
with the License. You may obtain a copy of the License at
9+
10+
http://www.apache.org/licenses/LICENSE-2.0
11+
12+
Unless required by applicable law or agreed to in writing, software
13+
distributed under the License is distributed on an "AS IS" BASIS,
14+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
See the License for the specific language governing permissions and
16+
limitations under the License.
17+
-->
18+
<assembly>
19+
<id>hadoop-sls</id>
20+
<formats>
21+
<format>dir</format>
22+
</formats>
23+
<includeBaseDirectory>false</includeBaseDirectory>
24+
25+
<fileSets>
26+
<fileSet>
27+
<directory>${basedir}/src/main/bin</directory>
28+
<outputDirectory>sls/bin</outputDirectory>
29+
<fileMode>0755</fileMode>
30+
</fileSet>
31+
<fileSet>
32+
<directory>${basedir}/src/main/html</directory>
33+
<outputDirectory>sls/html</outputDirectory>
34+
</fileSet>
35+
<fileSet>
36+
<directory>${basedir}/src/main/sample-conf</directory>
37+
<outputDirectory>sls/sample-conf</outputDirectory>
38+
</fileSet>
39+
<fileSet>
40+
<directory>${basedir}/src/main/data</directory>
41+
<outputDirectory>sls/sample-data</outputDirectory>
42+
</fileSet>
43+
</fileSets>
44+
45+
</assembly>

‎hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,17 @@
9393
<include>*-sources.jar</include>
9494
</includes>
9595
</fileSet>
96+
<fileSet>
97+
<directory>../hadoop-sls/target</directory>
98+
<outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
99+
<includes>
100+
<include>*-sources.jar</include>
101+
</includes>
102+
</fileSet>
103+
<fileSet>
104+
<directory>../hadoop-sls/target/hadoop-sls-${project.version}/sls</directory>
105+
<outputDirectory>/share/hadoop/${hadoop.component}/sls</outputDirectory>
106+
</fileSet>
96107
</fileSets>
97108
<dependencySets>
98109
<dependencySet>

‎hadoop-project/pom.xml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -729,6 +729,16 @@
729729
<artifactId>hsqldb</artifactId>
730730
<version>2.0.0</version>
731731
</dependency>
732+
<dependency>
733+
<groupId>com.codahale.metrics</groupId>
734+
<artifactId>metrics-core</artifactId>
735+
<version>3.0.0</version>
736+
</dependency>
737+
<dependency>
738+
<groupId>org.apache.hadoop</groupId>
739+
<artifactId>hadoop-sls</artifactId>
740+
<version>${project.version}</version>
741+
</dependency>
732742
</dependencies>
733743
</dependencyManagement>
734744

‎hadoop-project/src/site/site.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,7 @@
9595
<item name="Fair Scheduler" href="hadoop-yarn/hadoop-yarn-site/FairScheduler.html"/>
9696
<item name="Web Application Proxy" href="hadoop-yarn/hadoop-yarn-site/WebApplicationProxy.html"/>
9797
<item name="YARN Commands" href="hadoop-yarn/hadoop-yarn-site/YarnCommands.html"/>
98+
<item name="Scheduler Load Simulator" href="hadoop-sls/SchedulerLoadSimulator.html"/>
9899
</menu>
99100

100101
<menu name="YARN REST APIs" inherit="top">

‎hadoop-tools/hadoop-sls/README‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
Yarn Scheduler Load Simulator (SLS)
2+
3+
SLS is a stress and performance harness for the Yarn Resource Manager Scheduler
4+
that exercises the scheduler implementation simulating the cluster size and the
5+
applications load without having to have a cluster nor applications.
6+
7+
SLS runs a regular RM without RPC endpoints and uses a NodeManager and
8+
Application Manager simulators to send and receive events simulating cluster
9+
and application load behavior.
10+
11+
The size of the cluster and the application load is scripted in a configuration
12+
file.

‎hadoop-tools/hadoop-sls/dev-support/findbugs-exclude.xml‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
Licensed to the Apache Software Foundation (ASF) under one or more
4+
contributor license agreements. See the NOTICE file distributed with
5+
this work for additional information regarding copyright ownership.
6+
The ASF licenses this file to You under the Apache License, Version 2.0
7+
(the "License"); you may not use this file except in compliance with
8+
the License. You may obtain a copy of the License at
9+
10+
http://www.apache.org/licenses/LICENSE-2.0
11+
12+
Unless required by applicable law or agreed to in writing, software
13+
distributed under the License is distributed on an "AS IS" BASIS,
14+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
See the License for the specific language governing permissions and
16+
limitations under the License.
17+
-->
18+
<FindBugsFilter>
19+
20+
<!-- Ignore comparedTo, equals warnings -->
21+
<Match>
22+
<Class name="org.apache.hadoop.yarn.sls.scheduler.ContainerSimulator" />
23+
<Bug pattern="EQ_COMPARETO_USE_OBJECT_EQUALS" />
24+
</Match>
25+
26+
</FindBugsFilter>

‎hadoop-tools/hadoop-sls/pom.xml‎

Lines changed: 184 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,184 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
Licensed under the Apache License, Version 2.0 (the "License");
4+
you may not use this file except in compliance with the License.
5+
You may obtain a copy of the License at
6+
7+
http://www.apache.org/licenses/LICENSE-2.0
8+
9+
Unless required by applicable law or agreed to in writing, software
10+
distributed under the License is distributed on an "AS IS" BASIS,
11+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
See the License for the specific language governing permissions and
13+
limitations under the License. See accompanying LICENSE file.
14+
-->
15+
<project xmlns="http://maven.apache.org/POM/4.0.0"
16+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
17+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
18+
<modelVersion>4.0.0</modelVersion>
19+
<parent>
20+
<groupId>org.apache.hadoop</groupId>
21+
<artifactId>hadoop-project</artifactId>
22+
<version>2.3.0-SNAPSHOT</version>
23+
<relativePath>../../hadoop-project</relativePath>
24+
</parent>
25+
<groupId>org.apache.hadoop</groupId>
26+
<artifactId>hadoop-sls</artifactId>
27+
<version>2.3.0-SNAPSHOT</version>
28+
<description>Apache Hadoop Scheduler Load Simulator</description>
29+
<name>Apache Hadoop Scheduler Load Simulator</name>
30+
<packaging>jar</packaging>
31+
32+
<dependencies>
33+
<dependency>
34+
<groupId>junit</groupId>
35+
<artifactId>junit</artifactId>
36+
<scope>test</scope>
37+
</dependency>
38+
<dependency>
39+
<groupId>org.apache.hadoop</groupId>
40+
<artifactId>hadoop-client</artifactId>
41+
<scope>provided</scope>
42+
</dependency>
43+
<dependency>
44+
<groupId>org.apache.hadoop</groupId>
45+
<artifactId>hadoop-minicluster</artifactId>
46+
<scope>provided</scope>
47+
</dependency>
48+
<dependency>
49+
<groupId>org.apache.hadoop</groupId>
50+
<artifactId>hadoop-rumen</artifactId>
51+
</dependency>
52+
<dependency>
53+
<groupId>com.codahale.metrics</groupId>
54+
<artifactId>metrics-core</artifactId>
55+
<scope>compile</scope>
56+
</dependency>
57+
<dependency>
58+
<groupId>org.mortbay.jetty</groupId>
59+
<artifactId>jetty</artifactId>
60+
<scope>provided</scope>
61+
<exclusions>
62+
<exclusion>
63+
<groupId>org.mortbay.jetty</groupId>
64+
<artifactId>servlet-api</artifactId>
65+
</exclusion>
66+
</exclusions>
67+
</dependency>
68+
<dependency>
69+
<groupId>org.mortbay.jetty</groupId>
70+
<artifactId>jetty-util</artifactId>
71+
<scope>provided</scope>
72+
</dependency>
73+
</dependencies>
74+
75+
<build>
76+
<plugins>
77+
<plugin>
78+
<groupId>org.apache.maven.plugins</groupId>
79+
<artifactId>maven-source-plugin</artifactId>
80+
<configuration>
81+
<attach>true</attach>
82+
</configuration>
83+
<executions>
84+
<execution>
85+
<goals>
86+
<goal>jar</goal>
87+
</goals>
88+
</execution>
89+
</executions>
90+
</plugin>
91+
<plugin>
92+
<groupId>org.codehaus.mojo</groupId>
93+
<artifactId>findbugs-maven-plugin</artifactId>
94+
<configuration>
95+
<findbugsXmlOutput>true</findbugsXmlOutput>
96+
<xmlOutput>true</xmlOutput>
97+
<excludeFilterFile>${basedir}/dev-support/findbugs-exclude.xml</excludeFilterFile>
98+
<effort>Max</effort>
99+
</configuration>
100+
</plugin>
101+
<plugin>
102+
<groupId>org.apache.rat</groupId>
103+
<artifactId>apache-rat-plugin</artifactId>
104+
<configuration>
105+
<excludes>
106+
<exclude>src/main/data/2jobs2min-rumen-jh.json</exclude>
107+
<exclude>src/main/html/js/thirdparty/jquery.js</exclude>
108+
<exclude>src/main/html/js/thirdparty/d3-LICENSE</exclude>
109+
<exclude>src/main/html/js/thirdparty/d3.v3.js</exclude>
110+
<exclude>src/main/html/simulate.html.template</exclude>
111+
<exclude>src/main/html/simulate.info.html.template</exclude>
112+
<exclude>src/main/html/track.html.template</exclude>
113+
<exclude>src/test/resources/simulate.html.template</exclude>
114+
<exclude>src/test/resources/simulate.info.html.template</exclude>
115+
<exclude>src/test/resources/track.html.template</exclude>
116+
</excludes>
117+
</configuration>
118+
</plugin>
119+
</plugins>
120+
</build>
121+
122+
<profiles>
123+
<profile>
124+
<id>docs</id>
125+
<activation>
126+
<activeByDefault>false</activeByDefault>
127+
</activation>
128+
<build>
129+
<plugins>
130+
<plugin>
131+
<groupId>org.apache.maven.plugins</groupId>
132+
<artifactId>maven-site-plugin</artifactId>
133+
<executions>
134+
<execution>
135+
<phase>package</phase>
136+
<goals>
137+
<goal>site</goal>
138+
</goals>
139+
</execution>
140+
</executions>
141+
</plugin>
142+
</plugins>
143+
</build>
144+
</profile>
145+
<profile>
146+
<id>dist</id>
147+
<activation>
148+
<activeByDefault>false</activeByDefault>
149+
</activation>
150+
<build>
151+
<plugins>
152+
<plugin>
153+
<groupId>org.apache.maven.plugins</groupId>
154+
<artifactId>maven-assembly-plugin</artifactId>
155+
<dependencies>
156+
<dependency>
157+
<groupId>org.apache.hadoop</groupId>
158+
<artifactId>hadoop-assemblies</artifactId>
159+
<version>${project.version}</version>
160+
</dependency>
161+
</dependencies>
162+
<executions>
163+
<execution>
164+
<id>dist</id>
165+
<phase>prepare-package</phase>
166+
<goals>
167+
<goal>single</goal>
168+
</goals>
169+
<configuration>
170+
<appendAssemblyId>false</appendAssemblyId>
171+
<attach>false</attach>
172+
<finalName>${project.artifactId}-${project.version}</finalName>
173+
<descriptorRefs>
174+
<descriptorRef>hadoop-sls</descriptorRef>
175+
</descriptorRefs>
176+
</configuration>
177+
</execution>
178+
</executions>
179+
</plugin>
180+
</plugins>
181+
</build>
182+
</profile>
183+
</profiles>
184+
</project>

‎hadoop-tools/hadoop-sls/src/main/assemblies/sls.xml‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
<!--
2+
Licensed to the Apache Software Foundation (ASF) under one
3+
or more contributor license agreements. See the NOTICE file
4+
distributed with this work for additional information
5+
regarding copyright ownership. The ASF licenses this file
6+
to you under the Apache License, Version 2.0 (the
7+
"License"); you may not use this file except in compliance
8+
with the License. You may obtain a copy of the License at
9+
10+
http://www.apache.org/licenses/LICENSE-2.0
11+
12+
Unless required by applicable law or agreed to in writing, software
13+
distributed under the License is distributed on an "AS IS" BASIS,
14+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
See the License for the specific language governing permissions and
16+
limitations under the License.
17+
-->
18+
<assembly>
19+
<id>sls</id>
20+
<formats>
21+
<format>dir</format>
22+
<format>tar.gz</format>
23+
</formats>
24+
<includeBaseDirectory>false</includeBaseDirectory>
25+
26+
<fileSets>
27+
<fileSet>
28+
<directory>${basedir}/src/main/bin</directory>
29+
<outputDirectory>bin</outputDirectory>
30+
<fileMode>0755</fileMode>
31+
</fileSet>
32+
<fileSet>
33+
<directory>${basedir}/src/main/data</directory>
34+
<outputDirectory>sample-data</outputDirectory>
35+
</fileSet>
36+
<fileSet>
37+
<directory>${basedir}/src/main/html</directory>
38+
<outputDirectory>html</outputDirectory>
39+
</fileSet>
40+
<fileSet>
41+
<directory>${basedir}/src/main/sample-conf</directory>
42+
<outputDirectory>sample-conf</outputDirectory>
43+
</fileSet>
44+
</fileSets>
45+
46+
<dependencySets>
47+
<dependencySet>
48+
<outputDirectory>/lib</outputDirectory>
49+
<unpack>false</unpack>
50+
<scope>compile</scope>
51+
<useProjectArtifact>true</useProjectArtifact>
52+
</dependencySet>
53+
</dependencySets>
54+
55+
</assembly>

0 commit comments

Comments
 (0)