Commit dcb1718f authored by Joshua Ghost's avatar Joshua Ghost
Browse files

finish SGC and GAT

parent 390cd712
#!/bin/bash
python ./src/train_new.py \
--debug \
--datapath data// \
--seed 42 \
--dataset cora \
--type mutigcn \
--nhiddenlayer 1 \
--nbaseblocklayer 0 \
--hidden 128 \
--epoch 400 \
--lr 0.01 \
--weight_decay 0.005 \
--early_stopping 400 \
--sampling_percent 0.7 \
--dropout 0.8 \
--normalization FirstOrderGCN
python ./src/train_new.py \
--debug \
--datapath data// \
--seed 42 \
--dataset cora \
--type mutigcn \
--nhiddenlayer 1 \
--nbaseblocklayer 2 \
--hidden 128 \
--epoch 400 \
--lr 0.01 \
--weight_decay 0.005 \
--early_stopping 400 \
--sampling_percent 0.7 \
--dropout 0.8 \
--normalization FirstOrderGCN
python ./src/train_new.py \
--debug \
--datapath data// \
--seed 42 \
--dataset cora \
--type mutigcn \
--nhiddenlayer 1 \
--nbaseblocklayer 4 \
--hidden 128 \
--epoch 400 \
--lr 0.01 \
--weight_decay 0.005 \
--early_stopping 400 \
--sampling_percent 0.7 \
--dropout 0.8 \
--normalization FirstOrderGCN
python ./src/train_new.py \
--debug \
--datapath data// \
--seed 42 \
--dataset cora \
--type mutigcn \
--nhiddenlayer 1 \
--nbaseblocklayer 6 \
--hidden 128 \
--epoch 400 \
--lr 0.01 \
--weight_decay 0.005 \
--early_stopping 400 \
--sampling_percent 0.7 \
--dropout 0.8 \
--normalization FirstOrderGCN
python ./src/train_new.py \
--debug \
--datapath data// \
--seed 42 \
--dataset cora \
--type mutigcn \
--nhiddenlayer 1 \
--nbaseblocklayer 14 \
--hidden 128 \
--epoch 400 \
--lr 0.01 \
--weight_decay 0.005 \
--early_stopping 400 \
--sampling_percent 0.7 \
--dropout 0.8 \
--normalization FirstOrderGCN
\ No newline at end of file
...@@ -15,6 +15,4 @@ python ./src/train_new.py \ ...@@ -15,6 +15,4 @@ python ./src/train_new.py \
--early_stopping 400 \ --early_stopping 400 \
--sampling_percent 0.4 \ --sampling_percent 0.4 \
--dropout 0.8 \ --dropout 0.8 \
--normalization AugNormAdj --task_type semi \ --normalization AugNormAdj --task_type semi
\
#!/bin/bash
python ./src/train_new.py \
--debug \
--datapath data// \
--seed 42 \
--dataset cora \
--type mutigcn \
--nhiddenlayer 1 \
--nbaseblocklayer 0 \
--hidden 128 \
--epoch 400 \
--lr 0.007 \
--weight_decay 1e-05 \
--early_stopping 400 \
--sampling_percent 0.4 \
--dropout 0.8 \
--normalization AugNormAdj --task_type semi
python ./src/train_new.py \
--debug \
--datapath data// \
--seed 42 \
--dataset cora \
--type mutigcn \
--nhiddenlayer 1 \
--nbaseblocklayer 2 \
--hidden 128 \
--epoch 400 \
--lr 0.007 \
--weight_decay 1e-05 \
--early_stopping 400 \
--sampling_percent 0.4 \
--dropout 0.8 \
--normalization AugNormAdj --task_type semi
python ./src/train_new.py \
--debug \
--datapath data// \
--seed 42 \
--dataset cora \
--type mutigcn \
--nhiddenlayer 1 \
--nbaseblocklayer 4 \
--hidden 128 \
--epoch 400 \
--lr 0.007 \
--weight_decay 1e-05 \
--early_stopping 400 \
--sampling_percent 0.4 \
--dropout 0.8 \
--normalization AugNormAdj --task_type semi
python ./src/train_new.py \
--debug \
--datapath data// \
--seed 42 \
--dataset cora \
--type mutigcn \
--nhiddenlayer 1 \
--nbaseblocklayer 6 \
--hidden 128 \
--epoch 400 \
--lr 0.007 \
--weight_decay 1e-05 \
--early_stopping 400 \
--sampling_percent 0.4 \
--dropout 0.8 \
--normalization AugNormAdj --task_type semi
python ./src/train_new.py \
--debug \
--datapath data// \
--seed 42 \
--dataset cora \
--type mutigcn \
--nhiddenlayer 1 \
--nbaseblocklayer 14 \
--hidden 128 \
--epoch 400 \
--lr 0.007 \
--weight_decay 1e-05 \
--early_stopping 400 \
--sampling_percent 0.4 \
--dropout 0.8 \
--normalization AugNormAdj --task_type semi
\ No newline at end of file
#!/bin/bash
python ./src/train_new.py \
--debug \
--datapath data// \
--seed 42 \
--dataset cora \
--type sgc \
--nhiddenlayer 1 \
--nbaseblocklayer 2 \
--hidden 128 \
--epoch 400 \
--lr 0.007 \
--weight_decay 1e-05 \
--early_stopping 400 \
--sampling_percent 0.4 \
--dropout 0.8 \
--normalization AugNormAdj --task_type semi
...@@ -8,11 +8,10 @@ from torch import nn ...@@ -8,11 +8,10 @@ from torch import nn
import torch.nn.functional as F import torch.nn.functional as F
class BaseGraphConvolutionBS(Module):
class GraphConvolutionBS(Module): '''
""" Base class for GCN and SGC
GCN Layer with BN, Self-loop and Res connection. '''
"""
def __init__(self, in_features, out_features, activation=lambda x: x, withbn=True, withloop=True, bias=True, def __init__(self, in_features, out_features, activation=lambda x: x, withbn=True, withloop=True, bias=True,
res=False): res=False):
...@@ -26,7 +25,7 @@ class GraphConvolutionBS(Module): ...@@ -26,7 +25,7 @@ class GraphConvolutionBS(Module):
:param bias: enable bias. :param bias: enable bias.
:param res: enable res connections. :param res: enable res connections.
""" """
super(GraphConvolutionBS, self).__init__() super(BaseGraphConvolutionBS, self).__init__()
self.in_features = in_features self.in_features = in_features
self.out_features = out_features self.out_features = out_features
self.sigma = activation self.sigma = activation
...@@ -61,10 +60,7 @@ class GraphConvolutionBS(Module): ...@@ -61,10 +60,7 @@ class GraphConvolutionBS(Module):
if self.bias is not None: if self.bias is not None:
self.bias.data.uniform_(-stdv, stdv) self.bias.data.uniform_(-stdv, stdv)
def forward(self, input, adj): def postprocess(self, input, output):
support = torch.mm(input, self.weight)
output = torch.spmm(adj, support)
# Self-loop # Self-loop
if self.self_weight is not None: if self.self_weight is not None:
output = output + torch.mm(input, self.self_weight) output = output + torch.mm(input, self.self_weight)
...@@ -85,6 +81,36 @@ class GraphConvolutionBS(Module): ...@@ -85,6 +81,36 @@ class GraphConvolutionBS(Module):
+ str(self.in_features) + ' -> ' \ + str(self.in_features) + ' -> ' \
+ str(self.out_features) + ')' + str(self.out_features) + ')'
class SimpleGraphConvolutionBS(BaseGraphConvolutionBS):
def __init__(self, in_features, out_features, activation=lambda x: x, withbn=True, withloop=True, bias=True,
res=False):
super(SimpleGraphConvolutionBS, self).__init__(in_features, out_features,
activation,
withbn, withloop,
bias, res)
def forward(self, input, _):
output = torch.mm(input, self.weight)
return self.postprocess(input, output)
class GraphConvolutionBS(BaseGraphConvolutionBS):
def __init__(self, in_features, out_features,
activation=lambda x: x,
withbn=True, withloop=True,
bias=True, res=False):
super(GraphConvolutionBS, self).__init__(in_features, out_features,
activation,
withbn, withloop,
bias, res)
def forward(self, input, adj):
support = torch.mm(input, self.weight)
output = torch.spmm(adj, support)
return self.postprocess(input, output)
class GraphBaseBlock(Module): class GraphBaseBlock(Module):
""" """
The base block for Multi-layer GCN / ResGCN / Dense GCN The base block for Multi-layer GCN / ResGCN / Dense GCN
...@@ -92,7 +118,7 @@ class GraphBaseBlock(Module): ...@@ -92,7 +118,7 @@ class GraphBaseBlock(Module):
def __init__(self, in_features, out_features, nbaselayer, def __init__(self, in_features, out_features, nbaselayer,
withbn=True, withloop=True, activation=F.relu, dropout=True, withbn=True, withloop=True, activation=F.relu, dropout=True,
aggrmethod="concat", dense=False): aggrmethod="concat", dense=False, core='gcn'):
""" """
The base block for constructing DeepGCN model. The base block for constructing DeepGCN model.
:param in_features: the input feature dimension. :param in_features: the input feature dimension.
...@@ -117,7 +143,7 @@ class GraphBaseBlock(Module): ...@@ -117,7 +143,7 @@ class GraphBaseBlock(Module):
self.withbn = withbn self.withbn = withbn
self.withloop = withloop self.withloop = withloop
self.hiddenlayers = nn.ModuleList() self.hiddenlayers = nn.ModuleList()
self.__makehidden() self.__makehidden(core)
if self.aggrmethod == "concat" and dense == False: if self.aggrmethod == "concat" and dense == False:
self.out_features = in_features + out_features self.out_features = in_features + out_features
...@@ -132,14 +158,21 @@ class GraphBaseBlock(Module): ...@@ -132,14 +158,21 @@ class GraphBaseBlock(Module):
else: else:
raise NotImplementedError("The aggregation method only support 'concat','add' and 'nores'.") raise NotImplementedError("The aggregation method only support 'concat','add' and 'nores'.")
def __makehidden(self): def __makehidden(self, core):
# for i in xrange(self.nhiddenlayer): # for i in xrange(self.nhiddenlayer):
if core == 'gcn':
Layer = GraphConvolutionBS
elif core == 'sgc':
Layer = SimpleGraphConvolutionBS
else:
raise NotImplementedError
for i in range(self.nhiddenlayer): for i in range(self.nhiddenlayer):
if i == 0: if i == 0:
layer = GraphConvolutionBS(self.in_features, self.hiddendim, self.activation, self.withbn, layer = Layer(self.in_features, self.hiddendim, self.activation, self.withbn,
self.withloop) self.withloop)
else: else:
layer = GraphConvolutionBS(self.hiddendim, self.hiddendim, self.activation, self.withbn, self.withloop) layer = Layer(self.hiddendim, self.hiddendim, self.activation, self.withbn, self.withloop)
self.hiddenlayers.append(layer) self.hiddenlayers.append(layer)
def _doconcat(self, x, subx): def _doconcat(self, x, subx):
...@@ -177,14 +210,34 @@ class GraphBaseBlock(Module): ...@@ -177,14 +210,34 @@ class GraphBaseBlock(Module):
self.out_features) self.out_features)
class MultiLayerGCNBlock(Module): class MultiLayerBase(Module):
def __init__(self):
super(MultiLayerBase, self).__init__()
self.model = None
def forward(self, input, adj):
return self.model.forward(input, adj)
def get_outdim(self):
return self.model.get_outdim()
def __repr__(self):
return "%s %s (%d - [%d:%d] > %d)" % (self.__class__.__name__,
self.model.aggrmethod,
self.model.in_features,
self.model.hiddendim,
self.model.nhiddenlayer,
self.model.out_features)
class MultiLayerGCNBlock(MultiLayerBase):
""" """
Muti-Layer GCN with same hidden dimension. Muti-Layer GCN with same hidden dimension.
""" """
def __init__(self, in_features, out_features, nbaselayer, def __init__(self, in_features, out_features, nbaselayer,
withbn=True, withloop=True, activation=F.relu, dropout=True, withbn=True, withloop=True, activation=F.relu, dropout=True,
aggrmethod=None, dense=None): aggrmethod="nores", dense=None):
""" """
The multiple layer GCN block. The multiple layer GCN block.
:param in_features: the input feature dimension. :param in_features: the input feature dimension.
...@@ -205,22 +258,41 @@ class MultiLayerGCNBlock(Module): ...@@ -205,22 +258,41 @@ class MultiLayerGCNBlock(Module):
withloop=withloop, withloop=withloop,
activation=activation, activation=activation,
dropout=dropout, dropout=dropout,
dense=False, dense=dense,
aggrmethod="nores") aggrmethod=aggrmethod,
core='gcn')
def forward(self, input, adj):
return self.model.forward(input, adj)
def get_outdim(self):
return self.model.get_outdim()
def __repr__(self): class MultiLayerSGCBlock(MultiLayerBase):
return "%s %s (%d - [%d:%d] > %d)" % (self.__class__.__name__, """
self.aggrmethod, Multi-Layer Simplified Graph Convolutional Network (SGC) with same hidden dimension.
self.model.in_features, """
self.model.hiddendim, def __init__(self, in_features, out_features, nbaselayer,
self.model.nhiddenlayer, withbn=True, withloop=True, activation=F.relu, dropout=True,
self.model.out_features) aggrmethod=None, dense=None):
"""
The multiple layer GCN block.
:param in_features: the input feature dimension.
:param out_features: the hidden feature dimension.
:param nbaselayer: the number of layers in the base block.
:param withbn: using batch normalization in graph convolution.
:param withloop: using self feature modeling in graph convolution.
:param activation: the activation function, default is ReLu.
:param dropout: the dropout ratio.
:param aggrmethod: not applied.
:param dense: not applied.
"""
super(MultiLayerSGCBlock, self).__init__()
self.model = GraphBaseBlock(in_features=in_features,
out_features=out_features,
nbaselayer=nbaselayer,
withbn=withbn,
withloop=withloop,
activation=activation,
dropout=dropout,
dense=dense,
aggrmethod=aggrmethod,
core='sgc')
class ResGCNBlock(Module): class ResGCNBlock(Module):
...@@ -437,3 +509,50 @@ class Dense(Module): ...@@ -437,3 +509,50 @@ class Dense(Module):
return self.__class__.__name__ + ' (' \ return self.__class__.__name__ + ' (' \
+ str(self.in_features) + ' -> ' \ + str(self.in_features) + ' -> ' \
+ str(self.out_features) + ')' + str(self.out_features) + ')'
import torch
from torch import nn
import torch.nn.functional as F
class Attn_head(nn.Module):
def __init__(self, in_size: int, out_size: int, activation: nn.Module,
p_drop: float=0.0, attn_drop: float=0.0, residual: bool=False):
super(Attn_head, self).__init__()
if p_drop != 0.0:
self.drop_1 = nn.Dropout(p_drop)
self.drop_2 = nn.Dropout(p_drop)
self.attn_input = nn.Conv1d(in_channels=in_size, out_channels=out_size, kernel_size=1, bias=False)
self.attn_f1 = nn.Conv1d(in_channels=out_size, out_channels=1, kernel_size=1)
self.attn_f2 = nn.Conv1d(in_channels=out_size, out_channels=1, kernel_size=1)
if attn_drop != 0.0:
self.coef_drop = nn.Dropout(attn_drop)
self.val_bias = nn.Parameter(torch.FloatTensor([out_size]))
self.activation = activation
self.residual = residual
if self.residual:
self.res_conv = nn.Conv1d(in_channels=in_size, out_channels=out_size, kernel_size=1)
def forward(self, seq:torch.Tensor, bias_mat:torch.Tensor):
if hasattr(self, "drop_1"):
seq = self.l_drop(seq) # BNC
seq = seq.transpose(1, 2) # BCN
seq_fts = self.attn_input(seq) # BCN
f1 = self.attn_f1(seq_fts) # B1N
f2 = self.attn_f2(seq_fts) # B1N
logits = f1.transpose(1, 2) + f2 # BNN
coefs = F.softmax(F.leaky_relu(logits) + bias_mat) # BNN
if hasattr(self, 'coef_drop'):
coefs = self.coef_drop(coefs) # BNN
if hasattr(self, 'drop_2'):
seq_fts = self.drop_2(seq_fts) # BCN
vals = torch.matmul(seq_fts, coefs) # BCN
ret = (vals.transpose(1, 2) + self.val_bias).transpose(1, 2) # BCN
if self.residual:
if seq.shape[-2] != ret.shape[-2]:
ret = ret + self.res_conv(seq) # BCN
else:
ret = ret + seq
return self.activation(ret.transpose(1, 2)) # BNC
\ No newline at end of file
import math import math
import torch import torch
import torch.nn as nn import torch.nn as nn
from src.layers import *
from src.layers import Attn_head
import torch.nn.functional as F import torch.nn.functional as F
from layers import *
from torch.nn.parameter import Parameter
device = torch.device("cuda:0") device = torch.device("cuda:0")
...@@ -64,6 +64,8 @@ class GCNModel(nn.Module): ...@@ -64,6 +64,8 @@ class GCNModel(nn.Module):
self.BASEBLOCK = MultiLayerGCNBlock self.BASEBLOCK = MultiLayerGCNBlock
elif baseblock == "inceptiongcn": elif baseblock == "inceptiongcn":
self.BASEBLOCK = InecptionGCNBlock self.BASEBLOCK = InecptionGCNBlock
elif baseblock == 'sgc':
self.BASEBLOCK = MultiLayerSGCBlock
else: else:
raise NotImplementedError("Current baseblock %s is not supported." % (baseblock)) raise NotImplementedError("Current baseblock %s is not supported." % (baseblock))
if inputlayer == "gcn": if inputlayer == "gcn":
...@@ -135,31 +137,40 @@ class GCNModel(nn.Module): ...@@ -135,31 +137,40 @@ class GCNModel(nn.Module):
return x return x
# Modified GCN class GAT(Module):
class GCNFlatRes(nn.Module): def __init__(self, in_size, hidden_sizes, nbclasses,
""" attn_drop, p_drop,
(Legacy) n_heads, activation=F.elu, residual=False):
""" super(GAT, self).__init__()
def __init__(self, nfeat, nhid, nclass, withbn, nreslayer, dropout, mixmode=False): self.hidden_sizes = hidden_sizes
super(GCNFlatRes, self).__init__() self.n_heads = n_heads
self.attn_in = []
self.nreslayer = nreslayer for _ in range(self.n_heads[0]):
self.dropout = dropout self.attn_in.append(Attn_head(in_size=in_size, out_size=self.hidden_sizes[0],
self.ingc = GraphConvolution(nfeat, nhid, F.relu) activation=activation,
self.reslayer = GCFlatResBlock(nhid, nclass, nhid, nreslayer, dropout) p_drop=p_drop, attn_drop=attn_drop, residual=False))
self.reset_parameters() self.attns = []
for i in range(1, len(self.hidden_sizes)):
def reset_parameters(self): h_1 = []
# stdv = 1. / math.sqrt(self.attention.size(1)) for _ in range(self.n_heads[i]):
# self.attention.data.uniform_(-stdv, stdv) h_1.append(Attn_head(in_size=self.hidden_sizes[i - 1] * self.n_heads[i - 1],
# print(self.attention) out_size=self.hidden_sizes[i],
pass activation=activation,
p_drop=p_drop, attn_drop=attn_drop, residual=residual))
def forward(self, input, adj): self.attns.append(h_1)
x = self.ingc(input, adj) self.attn_out = []
x = F.dropout(x, self.dropout, training=self.training) for i in range(self.n_heads[-1]):
x = self.reslayer(x, adj) self.attn_out.append(Attn_head(in_size=self.hidden_sizes[-1] * self.n_heads[-2],
# x = F.dropout(x, self.dropout, training=self.training) out_size=nbclasses,
return F.log_softmax(x, dim=1) activation=lambda x: x,
p_drop=p_drop, attn_drop=attn_drop, residual=False))
def forward(self, inputs, bias_mat):
attns = [attn(seq=inputs, bias_mat=bias_mat) for attn in self.attn_in]
h_1 = torch.cat(attns, -1)
for i in range(1, len(self.hidden_sizes)):
h_1 = [attn(seq=h_1, bias_mat=bias_mat) for attn in self.attns[i - 1]]
h_1 = torch.cat(h_1, -1)
out = [attn(seq=h_1, bias_mat=bias_mat) for attn in self.attn_out]
logits = sum(out) / self.n_heads[-1]
return logits
...@@ -2,8 +2,8 @@ ...@@ -2,8 +2,8 @@
import numpy as np import numpy as np
import torch import torch
import scipy.sparse as sp import scipy.sparse as sp
from utils import data_loader, sparse_mx_to_torch_sparse_tensor from src.utils import data_loader, sparse_mx_to_torch_sparse_tensor