Commit 2f086b50 authored by Oleh Astappiev's avatar Oleh Astappiev
Browse files

Upload client as independent project

parents
Pipeline #436 failed with stages
# IDEA
.idea
target
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>shs-test-client</artifactId>
<groupId>de.l3s.shs</groupId>
<version>2.0.0</version>
<properties>
<manifest.mainClass>de.l3s.shs.shsclienttest.test.Runner</manifest.mainClass>
</properties>
<dependencies>
<dependency>
<groupId>de.siegmar</groupId>
<artifactId>fastcsv</artifactId>
<version>1.0.2</version>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.46</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>2.7</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>2.7</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.7.0</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<finalName>${project.name}</finalName>
<archive>
<manifest>
<mainClass>${manifest.mainClass}</mainClass>
</manifest>
</archive>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
<appendAssemblyId>false</appendAssemblyId>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
<component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_8">
<output url="file://$MODULE_DIR$/target/classes" />
<output-test url="file://$MODULE_DIR$/target/test-classes" />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/main/resources" type="java-resource" />
<excludeFolder url="file://$MODULE_DIR$/target" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="Maven: de.siegmar:fastcsv:1.0.2" level="project" />
<orderEntry type="library" name="Maven: mysql:mysql-connector-java:5.1.46" level="project" />
<orderEntry type="library" name="Maven: org.apache.logging.log4j:log4j-api:2.7" level="project" />
<orderEntry type="library" name="Maven: org.apache.logging.log4j:log4j-core:2.7" level="project" />
</component>
</module>
\ No newline at end of file
package de.l3s.shs.shsclienttest.client;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.SocketException;
import java.nio.ByteBuffer;
import java.nio.channels.SocketChannel;
import java.nio.charset.StandardCharsets;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
* @author vdkhoi
*/
public class Channel {
private String name; // may be null
private SocketChannel socket;
private ByteBuffer sendBuf;
private ByteBuffer recvBuf;
private ByteBuffer serBuf;
private int sendPos;
private int recvPos;
private int recvLen;
public boolean needRevert = true;
private static SocketChannel MakeSocket(String hostName, int port) {
SocketChannel s = null;
try {
s = SocketChannel.open(new InetSocketAddress(hostName, port));
} catch (IOException ex) {
Logger.getLogger(Channel.class.getName()).log(Level.SEVERE, null, ex);
}
return s;
}
public Channel(String name, int port) {
this(MakeSocket(name, port));
this.name = name;
}
public Channel(SocketChannel s) {
this.socket = s;
try {
this.socket.socket().setSoLinger(true, 1);
this.socket.socket().setSendBufferSize(1 << 23);
this.socket.socket().setReceiveBufferSize(1 << 23);
this.socket.socket().setTcpNoDelay(false);
this.serBuf = ByteBuffer.allocate(8);
this.recvBuf = ByteBuffer.allocate(this.socket.socket().getReceiveBufferSize());
this.sendBuf = ByteBuffer.allocate(this.socket.socket().getSendBufferSize());
this.sendPos = 0;
this.recvPos = 0;
this.recvLen = 0;
} catch (SocketException ex) {
Logger.getLogger(Channel.class.getName()).log(Level.SEVERE, null, ex);
}
}
public SocketChannel getSocket() {
return this.socket;
}
public String getName() {
return this.name;
}
public void WriteBoolean(Boolean x) {
if (this.sendPos + 1 < this.sendBuf.capacity()) {
this.sendBuf.put((byte) (x ? 1 : 0));
} else {
this.WriteNumBytes(this.serBuf, 1);
}
this.serBuf.rewind();
}
public Boolean ReadBoolean() {
if (this.recvPos + 1 <= this.recvLen) {
recvPos += 1;
return this.recvBuf.get(recvPos - 1) > 0;
} else {
this.ReadNumBytes(this.serBuf, 1);
return this.serBuf.get(0) > 0;
}
}
public void WriteInt32(int x) {
this.serBuf = this.serBuf.putInt(x);
RevertBuffer(serBuf, 0, 4);
if (this.sendPos + 4 < this.sendBuf.limit()) {
this.sendBuf.put(this.serBuf.array(), 0, 4);
this.sendPos += 4;
} else {
this.WriteNumBytes(this.serBuf, 4);
}
this.serBuf.rewind();
}
public int ReadInt32() {
if (this.recvPos + 4 <= this.recvLen) {
recvPos += 4;
RevertBuffer(this.recvBuf, recvPos - 4, 4);
return this.recvBuf.getInt(recvPos - 4);
} else {
this.serBuf.rewind();
int pos = this.serBuf.position();
this.ReadNumBytes(this.serBuf, 4);
RevertBuffer(serBuf, 0, 4);
int val = this.serBuf.getInt(pos);
this.serBuf.rewind();
return val;
}
}
public void WriteInt64(long x) {
this.serBuf = this.serBuf.putLong(x);
RevertBuffer(serBuf, 0, 8);
if (this.sendPos + 8 < this.sendBuf.limit()) {
this.sendBuf.put(this.serBuf.array(), 0, 8);
this.sendPos += 8;
} else {
this.WriteNumBytes(this.serBuf, 8);
}
this.serBuf.rewind();
}
public long ReadInt64() {
if (this.recvPos + 8 <= this.recvLen) {
recvPos += 8;
RevertBuffer(this.recvBuf, recvPos - 8, 8);
return this.recvBuf.getLong(recvPos - 8);
} else {
this.serBuf.rewind();
int pos = this.serBuf.position();
this.ReadNumBytes(this.serBuf, 8);
RevertBuffer(serBuf, 0, 8);
long val = this.serBuf.getLong(pos);
this.serBuf.rewind();
return val;
}
}
public void WriteUInt64(long x) {
WriteInt64(x);
}
public long ReadUInt64() {
return ReadInt64();
}
public void WriteUInt32(int x) {
WriteInt32(x);
}
public int ReadUInt32() {
return ReadInt32();
}
public void WriteString(String s) {
this.WriteBytes(ByteBuffer.wrap(s.getBytes(StandardCharsets.UTF_8)));
}
public String ReadString() {
ByteBuffer bytes = null;
try {
bytes = this.ReadBytes();
} catch (Exception ex) {
Logger.getLogger(Channel.class.getName()).log(Level.SEVERE, null, ex);
}
return bytes == null ? null : new String(bytes.array());
}
public void WriteBytes(ByteBuffer buf) {
if (buf == null) {
this.WriteInt32(-1);
} else {
int n = buf.capacity();
this.WriteInt32(buf.capacity());
int i = 0;
while (i < n) {
int cnt = Math.min(n, i + this.sendBuf.capacity() - this.sendPos);
this.sendBuf.put(buf.array(), 0, cnt);
i += cnt;
this.sendPos += cnt;
if (this.sendBuf.limit() == this.sendPos) {
this.FlushSendBuffer();
}
}
}
}
public ByteBuffer ReadBytes() throws Exception {
int n = this.ReadInt32();
if (n == -1) {
return null;
}
if (n > 0 && this.recvLen == 0) {
throw new Exception("stream too short");
}
ByteBuffer buf = ByteBuffer.allocate(n);
while (n > 0) {
if (this.recvPos == this.recvLen) {
this.FillRecvBuffer();
}
int cnt = Math.min(n, this.recvLen - this.recvPos);
buf.put(this.recvBuf.array(), this.recvPos, cnt);
this.recvPos += cnt;
n -= cnt;
}
return buf;
}
public void WriteNumBytes(ByteBuffer b, int num) {
while (num > 0) {
int cnt = Math.min(num, this.sendBuf.capacity() - this.sendPos);
this.sendBuf.put(b.array(), 0, cnt);
this.sendPos += cnt;
num -= cnt;
if (this.sendBuf.capacity() == this.sendPos) {
this.FlushSendBuffer();
}
}
}
public void ReadNumBytes(ByteBuffer b, int num) {
int cnt = 0;
try {
while (num > 0) {
if (this.recvPos >= this.recvLen) {
this.FillRecvBuffer();
}
if (this.recvLen < 0) return;
cnt = Math.min(num, this.recvLen - this.recvPos);
b = b.put(this.recvBuf.array(), this.recvPos, cnt);
this.recvPos += cnt;
num -= cnt;
}
}
catch (IndexOutOfBoundsException e) {
System.out.println("" + this.recvBuf.array().length + " : " + this.recvPos + " : " + cnt + " : " + this.recvLen + " : " + this.recvBuf.remaining());
throw e;
}
}
public void Close() {
try {
this.FlushSendBuffer();
this.socket.socket().shutdownInput();
this.socket.socket().shutdownOutput();
this.socket.socket().close();
} catch (IOException se) {
Logger.getLogger(Channel.class.getName()).log(Level.SEVERE, null, se);
}
this.socket = null;
}
public void Flush() {
try {
// Disable Nagle to ensure flushed data is sent right away
this.socket.socket().setTcpNoDelay(true);
this.FlushSendBuffer();
// Re-enable Nagle to fully utilize MTUs
this.socket.socket().setTcpNoDelay(false);
} catch (SocketException ex) {
Logger.getLogger(Channel.class.getName()).log(Level.SEVERE, null, ex);
}
}
private void FlushSendBuffer() {
try {
this.socket.write(ByteBuffer.wrap(this.sendBuf.array(), 0, this.sendPos));
this.sendBuf.rewind();
this.sendPos = 0;
} catch (IOException ex) {
Logger.getLogger(Channel.class.getName()).log(Level.SEVERE, null, ex);
}
}
private void FillRecvBuffer() {
this.recvPos = 0;
this.recvBuf = (ByteBuffer)this.recvBuf.rewind();
try {
this.recvLen = this.socket.read(this.recvBuf);
if (this.recvLen == 0) {
throw new SocketException();
}
} catch (Exception ex) {
ex.printStackTrace();
Logger.getLogger(Channel.class.getName()).log(Level.SEVERE, null, ex);
}
}
private void RevertBuffer(ByteBuffer buf, int offset, int num_revert) {
if (!this.needRevert) {
return;
}
byte[] bytes = buf.array();
for (int left = offset, right = offset + num_revert - 1; left < right; ++left, --right) {
byte temp = bytes[left];
bytes[left] = bytes[right];
bytes[right] = temp;
}
}
}
package de.l3s.shs.shsclienttest.client;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
/**
* @author vdkhoi
*/
public class Demo {
public static void main(String[] agrs) {
String[] sample = {"bild.de/", "angela-merkel.de/"};
String firstTime = "2013-01-01";
String lastTime = "2013-02-15";
System.out.println("Start connection...");
SHSClient client = new SHSClient();
System.out.println("UrlToUid: " + sample[0]);
long uid = client.getUid(sample[0]);
System.out.println("UrlToUid=" + uid);
// long uid = client.getUid("heise.de/");
// final long startTime = System.currentTimeMillis();
// Map<Long, List<Long>> outgoingLinksBild = client.getAllCapturesGroupByLinks(uid, Direction.FORWARD);
// final long elapsedTime = System.currentTimeMillis() - startTime;
// System.out.println("Total links " + outgoingLinksBild.size());
// System.out.println("Elapsed time: " + elapsedTime + " ms.");
// // System.out.println("Found UID: " + uid);
//
// long uid = client.getUid("bild.de/");
// long uid2 = client.getUid("angela-merkel.de/");
System.out.println("UrlToUid: " + Arrays.toString(sample));
long[] uids = client.getUid(sample);
for (long uid1 : uids) {
System.out.println(uid1);
}
System.out.println("UrlToUid done.");
System.out.println("UidToUrl: " + uid);
String url = client.getUrl(uid);
System.out.println("UidToUrl=" + url);
System.out.println("UidToUrl: " + Arrays.toString(uids));
String[] urls = client.getUrl(uids);
for (String url1 : urls) {
System.out.println(url1);
}
System.out.println("UidToUrl done.");
System.out.println("GetTemporalLinks: forward for " + sample[1]);
urls = client.getTemporalLinks(sample[1], firstTime, lastTime, Direction.FORWARD);
System.out.println(Arrays.toString(urls));
System.out.println("GetTemporalLinks: backward for " + sample[1]);
urls = client.getTemporalLinks(sample[1], firstTime, lastTime, Direction.BACKWARD);
System.out.println(Arrays.toString(urls));
System.out.println("GetTemporalLinks done.");
System.out.println("GetTemporalLinks: for " + Arrays.toString(sample));
String[][] outLinksMap = client.getTemporalLinks(sample, firstTime, lastTime, Direction.FORWARD);
long count = 0;
for (String[] outLinks : outLinksMap) {
count += outLinks.length;
System.out.println(Arrays.toString(outLinks));
}
System.out.println("Found " + outLinksMap.length + " links for " + sample.length + " urls.");
System.out.println("GetTemporalLinks done.");
System.out.println("GetTemporalLinks: for " + uid);
long[] uids_1 = client.getTemporalLinks(uid, firstTime, lastTime, Direction.FORWARD);
for (long l : uids_1) {
System.out.println(l);
}
System.out.println("GetTemporalLinks done.");
System.out.println("GetTemporalLinks: for " + Arrays.toString(uids));
long[][] outLinks_1 = client.getTemporalLinks(uids, firstTime, lastTime, Direction.BACKWARD);
long count2 = 0;
for (long[] outLinks : outLinks_1) {
count2 += outLinks.length;
System.out.println(Arrays.toString(outLinks));
}
System.out.println("Found " + count2 + " links for " + outLinks_1.length + " uids.");
System.out.println("GetTemporalLinks done.");
long bildUid = client.getUid("bild.de/");
System.out.println("getAllCaptures...");
long[][] outLinksBild = client.getAllCaptures(bildUid, Direction.FORWARD);
System.out.println("Found " + outLinksBild.length + " links ");
for (long[] longs : outLinksBild) {
System.out.print("Time: " + TimeUtils.getFormattedDate(SHSClient.getTimestampFromDays(longs[0])) + "\t");
for (int j = 1; j < longs.length; j++) {
System.out.print(longs[j] + " ");
}
System.out.println();
}
System.out.println("getAllCaptures done.");
System.out.println("getAllCapturesGroupByLinks for " + bildUid);
Map<Long, List<Long>> outgoingLinksBild = client.getAllCapturesGroupByLinks(bildUid, Direction.FORWARD);
System.out.println("Found " + outgoingLinksBild.size() + " links");
for (Map.Entry<Long, List<Long>> entry : outgoingLinksBild.entrySet()) {
System.out.print("Uid: " + entry.getKey() + "\t");
for (Long value : entry.getValue()) {
System.out.print(TimeUtils.getFormattedDate(value) + " ");
}
System.out.println();
}
System.out.println("getAllCapturesGroupByLinks done.");
client.close();
}
}
\ No newline at end of file
package de.l3s.shs.shsclienttest.client;
public enum Direction {
FORWARD(0),
BACKWARD(1),
/**
* Alias for FORWARD
*/
OUTGOING(0),
/**
* Alias for BACKWARD
*/
INCOMING(1);
private final int value;
Direction(int value) {
this.value = value;
}
public int getValue() {
return value;
}
}
package de.l3s.shs.shsclienttest.client;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
/**
* @author vdkhoi
*/
public class LoadPartitions {
public static void main(String[] agrs) throws Exception{
// String[] sample = { "aev-forum.de/spiele-tipps-f81/2-pre-play-off-spiel-2008-09-grizzly-t32902/seite4.html",
// "adfarm.mediaplex.com/ad/ck/707-1170-4560-4/?redirectenter&partner=35087&loc=http://cgi.ebay.de/ws/ebayisapi.dll%3fviewitem%26item=2050588731",
// "ailingen.de/test/index.php?module=postcalendar&func=view&tplview=&viewtype=day&date=20050106&pc_username=&pc_category=&pc_topic=&print=1",
// "agriaffaires.it/usato/trattori-agricoli/835995/massey-ferguson-2640.html",
// "adsdeck.de/autos/go/6449307?source=70&robot=1"