Commit dcb1718f authored by Joshua Ghost's avatar Joshua Ghost
Browse files

finish SGC and GAT

parent 390cd712
#!/bin/bash
python ./src/train_new.py \
--debug \
--datapath data// \
--seed 42 \
--dataset cora \
--type mutigcn \
--nhiddenlayer 1 \
--nbaseblocklayer 0 \
--hidden 128 \
--epoch 400 \
--lr 0.01 \
--weight_decay 0.005 \
--early_stopping 400 \
--sampling_percent 0.7 \
--dropout 0.8 \
--normalization FirstOrderGCN
python ./src/train_new.py \
--debug \
--datapath data// \
--seed 42 \
--dataset cora \
--type mutigcn \
--nhiddenlayer 1 \
--nbaseblocklayer 2 \
--hidden 128 \
--epoch 400 \
--lr 0.01 \
--weight_decay 0.005 \
--early_stopping 400 \
--sampling_percent 0.7 \
--dropout 0.8 \
--normalization FirstOrderGCN
python ./src/train_new.py \
--debug \
--datapath data// \
--seed 42 \
--dataset cora \
--type mutigcn \
--nhiddenlayer 1 \
--nbaseblocklayer 4 \
--hidden 128 \
--epoch 400 \
--lr 0.01 \
--weight_decay 0.005 \
--early_stopping 400 \
--sampling_percent 0.7 \
--dropout 0.8 \
--normalization FirstOrderGCN
python ./src/train_new.py \
--debug \
--datapath data// \
--seed 42 \
--dataset cora \
--type mutigcn \
--nhiddenlayer 1 \
--nbaseblocklayer 6 \
--hidden 128 \
--epoch 400 \
--lr 0.01 \
--weight_decay 0.005 \
--early_stopping 400 \
--sampling_percent 0.7 \
--dropout 0.8 \
--normalization FirstOrderGCN
python ./src/train_new.py \
--debug \
--datapath data// \
--seed 42 \
--dataset cora \
--type mutigcn \
--nhiddenlayer 1 \
--nbaseblocklayer 14 \
--hidden 128 \
--epoch 400 \
--lr 0.01 \
--weight_decay 0.005 \
--early_stopping 400 \
--sampling_percent 0.7 \
--dropout 0.8 \
--normalization FirstOrderGCN
\ No newline at end of file
......@@ -15,6 +15,4 @@ python ./src/train_new.py \
--early_stopping 400 \
--sampling_percent 0.4 \
--dropout 0.8 \
--normalization AugNormAdj --task_type semi \
\
--normalization AugNormAdj --task_type semi
#!/bin/bash
python ./src/train_new.py \
--debug \
--datapath data// \
--seed 42 \
--dataset cora \
--type mutigcn \
--nhiddenlayer 1 \
--nbaseblocklayer 0 \
--hidden 128 \
--epoch 400 \
--lr 0.007 \
--weight_decay 1e-05 \
--early_stopping 400 \
--sampling_percent 0.4 \
--dropout 0.8 \
--normalization AugNormAdj --task_type semi
python ./src/train_new.py \
--debug \
--datapath data// \
--seed 42 \
--dataset cora \
--type mutigcn \
--nhiddenlayer 1 \
--nbaseblocklayer 2 \
--hidden 128 \
--epoch 400 \
--lr 0.007 \
--weight_decay 1e-05 \
--early_stopping 400 \
--sampling_percent 0.4 \
--dropout 0.8 \
--normalization AugNormAdj --task_type semi
python ./src/train_new.py \
--debug \
--datapath data// \
--seed 42 \
--dataset cora \
--type mutigcn \
--nhiddenlayer 1 \
--nbaseblocklayer 4 \
--hidden 128 \
--epoch 400 \
--lr 0.007 \
--weight_decay 1e-05 \
--early_stopping 400 \
--sampling_percent 0.4 \
--dropout 0.8 \
--normalization AugNormAdj --task_type semi
python ./src/train_new.py \
--debug \
--datapath data// \
--seed 42 \
--dataset cora \
--type mutigcn \
--nhiddenlayer 1 \
--nbaseblocklayer 6 \
--hidden 128 \
--epoch 400 \
--lr 0.007 \
--weight_decay 1e-05 \
--early_stopping 400 \
--sampling_percent 0.4 \
--dropout 0.8 \
--normalization AugNormAdj --task_type semi
python ./src/train_new.py \
--debug \
--datapath data// \
--seed 42 \
--dataset cora \
--type mutigcn \
--nhiddenlayer 1 \
--nbaseblocklayer 14 \
--hidden 128 \
--epoch 400 \
--lr 0.007 \
--weight_decay 1e-05 \
--early_stopping 400 \
--sampling_percent 0.4 \
--dropout 0.8 \
--normalization AugNormAdj --task_type semi
\ No newline at end of file
#!/bin/bash
python ./src/train_new.py \
--debug \
--datapath data// \
--seed 42 \
--dataset cora \
--type sgc \
--nhiddenlayer 1 \
--nbaseblocklayer 2 \
--hidden 128 \
--epoch 400 \
--lr 0.007 \
--weight_decay 1e-05 \
--early_stopping 400 \
--sampling_percent 0.4 \
--dropout 0.8 \
--normalization AugNormAdj --task_type semi
......@@ -8,11 +8,10 @@ from torch import nn
import torch.nn.functional as F
class GraphConvolutionBS(Module):
"""
GCN Layer with BN, Self-loop and Res connection.
"""
class BaseGraphConvolutionBS(Module):
'''
Base class for GCN and SGC
'''
def __init__(self, in_features, out_features, activation=lambda x: x, withbn=True, withloop=True, bias=True,
res=False):
......@@ -26,7 +25,7 @@ class GraphConvolutionBS(Module):
:param bias: enable bias.
:param res: enable res connections.
"""
super(GraphConvolutionBS, self).__init__()
super(BaseGraphConvolutionBS, self).__init__()
self.in_features = in_features
self.out_features = out_features
self.sigma = activation
......@@ -61,10 +60,7 @@ class GraphConvolutionBS(Module):
if self.bias is not None:
self.bias.data.uniform_(-stdv, stdv)
def forward(self, input, adj):
support = torch.mm(input, self.weight)
output = torch.spmm(adj, support)
def postprocess(self, input, output):
# Self-loop
if self.self_weight is not None:
output = output + torch.mm(input, self.self_weight)
......@@ -85,6 +81,36 @@ class GraphConvolutionBS(Module):
+ str(self.in_features) + ' -> ' \
+ str(self.out_features) + ')'
class SimpleGraphConvolutionBS(BaseGraphConvolutionBS):
def __init__(self, in_features, out_features, activation=lambda x: x, withbn=True, withloop=True, bias=True,
res=False):
super(SimpleGraphConvolutionBS, self).__init__(in_features, out_features,
activation,
withbn, withloop,
bias, res)
def forward(self, input, _):
output = torch.mm(input, self.weight)
return self.postprocess(input, output)
class GraphConvolutionBS(BaseGraphConvolutionBS):
def __init__(self, in_features, out_features,
activation=lambda x: x,
withbn=True, withloop=True,
bias=True, res=False):
super(GraphConvolutionBS, self).__init__(in_features, out_features,
activation,
withbn, withloop,
bias, res)
def forward(self, input, adj):
support = torch.mm(input, self.weight)
output = torch.spmm(adj, support)
return self.postprocess(input, output)
class GraphBaseBlock(Module):
"""
The base block for Multi-layer GCN / ResGCN / Dense GCN
......@@ -92,7 +118,7 @@ class GraphBaseBlock(Module):
def __init__(self, in_features, out_features, nbaselayer,
withbn=True, withloop=True, activation=F.relu, dropout=True,
aggrmethod="concat", dense=False):
aggrmethod="concat", dense=False, core='gcn'):
"""
The base block for constructing DeepGCN model.
:param in_features: the input feature dimension.
......@@ -117,7 +143,7 @@ class GraphBaseBlock(Module):
self.withbn = withbn
self.withloop = withloop
self.hiddenlayers = nn.ModuleList()
self.__makehidden()
self.__makehidden(core)
if self.aggrmethod == "concat" and dense == False:
self.out_features = in_features + out_features
......@@ -132,14 +158,21 @@ class GraphBaseBlock(Module):
else:
raise NotImplementedError("The aggregation method only support 'concat','add' and 'nores'.")
def __makehidden(self):
def __makehidden(self, core):
# for i in xrange(self.nhiddenlayer):
if core == 'gcn':
Layer = GraphConvolutionBS
elif core == 'sgc':
Layer = SimpleGraphConvolutionBS
else:
raise NotImplementedError
for i in range(self.nhiddenlayer):
if i == 0:
layer = GraphConvolutionBS(self.in_features, self.hiddendim, self.activation, self.withbn,
layer = Layer(self.in_features, self.hiddendim, self.activation, self.withbn,
self.withloop)
else:
layer = GraphConvolutionBS(self.hiddendim, self.hiddendim, self.activation, self.withbn, self.withloop)
layer = Layer(self.hiddendim, self.hiddendim, self.activation, self.withbn, self.withloop)
self.hiddenlayers.append(layer)
def _doconcat(self, x, subx):
......@@ -177,14 +210,34 @@ class GraphBaseBlock(Module):
self.out_features)
class MultiLayerGCNBlock(Module):
class MultiLayerBase(Module):
def __init__(self):
super(MultiLayerBase, self).__init__()
self.model = None
def forward(self, input, adj):
return self.model.forward(input, adj)
def get_outdim(self):
return self.model.get_outdim()
def __repr__(self):
return "%s %s (%d - [%d:%d] > %d)" % (self.__class__.__name__,
self.model.aggrmethod,
self.model.in_features,
self.model.hiddendim,
self.model.nhiddenlayer,
self.model.out_features)
class MultiLayerGCNBlock(MultiLayerBase):
"""
Muti-Layer GCN with same hidden dimension.
"""
def __init__(self, in_features, out_features, nbaselayer,
withbn=True, withloop=True, activation=F.relu, dropout=True,
aggrmethod=None, dense=None):
aggrmethod="nores", dense=None):
"""
The multiple layer GCN block.
:param in_features: the input feature dimension.
......@@ -205,22 +258,41 @@ class MultiLayerGCNBlock(Module):
withloop=withloop,
activation=activation,
dropout=dropout,
dense=False,
aggrmethod="nores")
def forward(self, input, adj):
return self.model.forward(input, adj)
dense=dense,
aggrmethod=aggrmethod,
core='gcn')
def get_outdim(self):
return self.model.get_outdim()
def __repr__(self):
return "%s %s (%d - [%d:%d] > %d)" % (self.__class__.__name__,
self.aggrmethod,
self.model.in_features,
self.model.hiddendim,
self.model.nhiddenlayer,
self.model.out_features)
class MultiLayerSGCBlock(MultiLayerBase):
"""
Multi-Layer Simplified Graph Convolutional Network (SGC) with same hidden dimension.
"""
def __init__(self, in_features, out_features, nbaselayer,
withbn=True, withloop=True, activation=F.relu, dropout=True,
aggrmethod=None, dense=None):
"""
The multiple layer GCN block.
:param in_features: the input feature dimension.
:param out_features: the hidden feature dimension.
:param nbaselayer: the number of layers in the base block.
:param withbn: using batch normalization in graph convolution.
:param withloop: using self feature modeling in graph convolution.
:param activation: the activation function, default is ReLu.
:param dropout: the dropout ratio.
:param aggrmethod: not applied.
:param dense: not applied.
"""
super(MultiLayerSGCBlock, self).__init__()
self.model = GraphBaseBlock(in_features=in_features,
out_features=out_features,
nbaselayer=nbaselayer,
withbn=withbn,
withloop=withloop,
activation=activation,
dropout=dropout,
dense=dense,
aggrmethod=aggrmethod,
core='sgc')
class ResGCNBlock(Module):
......@@ -437,3 +509,50 @@ class Dense(Module):
return self.__class__.__name__ + ' (' \
+ str(self.in_features) + ' -> ' \
+ str(self.out_features) + ')'
import torch
from torch import nn
import torch.nn.functional as F
class Attn_head(nn.Module):
def __init__(self, in_size: int, out_size: int, activation: nn.Module,
p_drop: float=0.0, attn_drop: float=0.0, residual: bool=False):
super(Attn_head, self).__init__()
if p_drop != 0.0:
self.drop_1 = nn.Dropout(p_drop)
self.drop_2 = nn.Dropout(p_drop)
self.attn_input = nn.Conv1d(in_channels=in_size, out_channels=out_size, kernel_size=1, bias=False)
self.attn_f1 = nn.Conv1d(in_channels=out_size, out_channels=1, kernel_size=1)
self.attn_f2 = nn.Conv1d(in_channels=out_size, out_channels=1, kernel_size=1)
if attn_drop != 0.0:
self.coef_drop = nn.Dropout(attn_drop)
self.val_bias = nn.Parameter(torch.FloatTensor([out_size]))
self.activation = activation
self.residual = residual
if self.residual:
self.res_conv = nn.Conv1d(in_channels=in_size, out_channels=out_size, kernel_size=1)
def forward(self, seq:torch.Tensor, bias_mat:torch.Tensor):
if hasattr(self, "drop_1"):
seq = self.l_drop(seq) # BNC
seq = seq.transpose(1, 2) # BCN
seq_fts = self.attn_input(seq) # BCN
f1 = self.attn_f1(seq_fts) # B1N
f2 = self.attn_f2(seq_fts) # B1N
logits = f1.transpose(1, 2) + f2 # BNN
coefs = F.softmax(F.leaky_relu(logits) + bias_mat) # BNN
if hasattr(self, 'coef_drop'):
coefs = self.coef_drop(coefs) # BNN
if hasattr(self, 'drop_2'):
seq_fts = self.drop_2(seq_fts) # BCN
vals = torch.matmul(seq_fts, coefs) # BCN
ret = (vals.transpose(1, 2) + self.val_bias).transpose(1, 2) # BCN
if self.residual:
if seq.shape[-2] != ret.shape[-2]:
ret = ret + self.res_conv(seq) # BCN
else:
ret = ret + seq
return self.activation(ret.transpose(1, 2)) # BNC
\ No newline at end of file
import math
import torch
import torch.nn as nn
from src.layers import *
from src.layers import Attn_head
import torch.nn.functional as F
from layers import *
from torch.nn.parameter import Parameter
device = torch.device("cuda:0")
......@@ -64,6 +64,8 @@ class GCNModel(nn.Module):
self.BASEBLOCK = MultiLayerGCNBlock
elif baseblock == "inceptiongcn":
self.BASEBLOCK = InecptionGCNBlock
elif baseblock == 'sgc':
self.BASEBLOCK = MultiLayerSGCBlock
else:
raise NotImplementedError("Current baseblock %s is not supported." % (baseblock))
if inputlayer == "gcn":
......@@ -135,31 +137,40 @@ class GCNModel(nn.Module):
return x
# Modified GCN
class GCNFlatRes(nn.Module):
"""
(Legacy)
"""
def __init__(self, nfeat, nhid, nclass, withbn, nreslayer, dropout, mixmode=False):
super(GCNFlatRes, self).__init__()
self.nreslayer = nreslayer
self.dropout = dropout
self.ingc = GraphConvolution(nfeat, nhid, F.relu)
self.reslayer = GCFlatResBlock(nhid, nclass, nhid, nreslayer, dropout)
self.reset_parameters()
def reset_parameters(self):
# stdv = 1. / math.sqrt(self.attention.size(1))
# self.attention.data.uniform_(-stdv, stdv)
# print(self.attention)
pass
def forward(self, input, adj):
x = self.ingc(input, adj)
x = F.dropout(x, self.dropout, training=self.training)
x = self.reslayer(x, adj)
# x = F.dropout(x, self.dropout, training=self.training)
return F.log_softmax(x, dim=1)
class GAT(Module):
def __init__(self, in_size, hidden_sizes, nbclasses,
attn_drop, p_drop,
n_heads, activation=F.elu, residual=False):
super(GAT, self).__init__()
self.hidden_sizes = hidden_sizes
self.n_heads = n_heads
self.attn_in = []
for _ in range(self.n_heads[0]):
self.attn_in.append(Attn_head(in_size=in_size, out_size=self.hidden_sizes[0],
activation=activation,
p_drop=p_drop, attn_drop=attn_drop, residual=False))
self.attns = []
for i in range(1, len(self.hidden_sizes)):
h_1 = []
for _ in range(self.n_heads[i]):
h_1.append(Attn_head(in_size=self.hidden_sizes[i - 1] * self.n_heads[i - 1],
out_size=self.hidden_sizes[i],
activation=activation,
p_drop=p_drop, attn_drop=attn_drop, residual=residual))
self.attns.append(h_1)
self.attn_out = []
for i in range(self.n_heads[-1]):
self.attn_out.append(Attn_head(in_size=self.hidden_sizes[-1] * self.n_heads[-2],
out_size=nbclasses,
activation=lambda x: x,
p_drop=p_drop, attn_drop=attn_drop, residual=False))
def forward(self, inputs, bias_mat):
attns = [attn(seq=inputs, bias_mat=bias_mat) for attn in self.attn_in]
h_1 = torch.cat(attns, -1)
for i in range(1, len(self.hidden_sizes)):
h_1 = [attn(seq=h_1, bias_mat=bias_mat) for attn in self.attns[i - 1]]
h_1 = torch.cat(h_1, -1)
out = [attn(seq=h_1, bias_mat=bias_mat) for attn in self.attn_out]
logits = sum(out) / self.n_heads[-1]
return logits
......@@ -2,8 +2,8 @@
import numpy as np
import torch
import scipy.sparse as sp
from utils import data_loader, sparse_mx_to_torch_sparse_tensor
from normalization import fetch_normalization
from src.utils import data_loader, sparse_mx_to_torch_sparse_tensor
from src.normalization import fetch_normalization
class Sampler:
"""Sampling the input graph data."""
......
......@@ -10,17 +10,17 @@ import torch.nn.functional as F
import torch.optim as optim
from tensorboardX import SummaryWriter
from earlystopping import EarlyStopping
from sample import Sampler
from metric import accuracy, roc_auc_compute_fn
from src.earlystopping import EarlyStopping
from src.sample import Sampler
from src.metric import accuracy, roc_auc_compute_fn
# from deepgcn.utils import load_data, accuracy
# from deepgcn.models import GCN
from metric import accuracy
from utils import load_citation, load_reddit_data
from models import *
from earlystopping import EarlyStopping
from sample import Sampler
from src.metric import accuracy
from src.utils import load_citation, load_reddit_data
from src.models import *
from src.earlystopping import EarlyStopping
from src.sample import Sampler
# Training settings
parser = argparse.ArgumentParser()
......@@ -76,7 +76,8 @@ parser.add_argument("--nbaseblocklayer", type=int, default=1,
help="The number of layers in each baseblock")
parser.add_argument("--aggrmethod", default="default",
help="The aggrmethod for the layer aggreation. The options includes add and concat. Only valid in resgcn, densegcn and inecptiongcn")
parser.add_argument("--task_type", default="full", help="The node classification task type (full and semi). Only valid for cora, citeseer and pubmed dataset.")
parser.add_argument("--task_type", default="full",
help="The node classification task type (full and semi). Only valid for cora, citeseer and pubmed dataset.")
args = parser.parse_args()
if args.debug:
......@@ -127,7 +128,8 @@ model = GCNModel(nfeat=nfeat,
withloop=args.withloop,
aggrmethod=args.aggrmethod,
mixmode=args.mixmode)
print(args.aggrmethod)
print(model)
optimizer = optim.Adam(model.parameters(),
lr=args.lr, weight_decay=args.weight_decay)
......@@ -159,6 +161,7 @@ if args.no_tensorboard is False:
comment=f"-dataset_{args.dataset}-type_{args.type}"
)
def get_lr(optimizer):
for param_group in optimizer.param_groups:
return param_group['lr']
......@@ -248,7 +251,7 @@ for epoch in range(args.epochs):
train_adj = train_adj.cuda()
sampling_t = time.time() - sampling_t