Skip to content

Commit

Permalink
refractor code
Browse files Browse the repository at this point in the history
  • Loading branch information
@zhihou7
zhihou7 committed Dec 5, 2022
1 parent a23c77a commit 85e15d3
Show file tree
Hide file tree
Showing 15 changed files with 1,453 additions and 19 deletions.
2 changes: 1 addition & 1 deletion GETTING_STARTED_SCL.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ python tools/Train_ATL_HICO.py --model VCL_union_batch_large2_ml5_def1_vloss2_VE

Here, AF713 is just a simple symbol to represent concept discovery with self-training,
'VERB' means we optimize verb category rather than HOI category,
'affordance' means online concept discovery. We will revise this to self-training in the final version. Baseline of HICO-DET is
'affordance' means online concept discovery. Baseline of HICO-DET is

```shell
python tools/Train_ATL_HICO.py --model VCL_union_batch_large2_ml5_def1_vloss2_VERB_l2_aug5_3_x5new_res101_affordance_9 --num_iteration 3000000
Expand Down
4 changes: 2 additions & 2 deletions lib/models/test_VCOCO.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@

from ult.config import cfg
from ult.timer import Timer
from ult.ult import Get_next_sp, Get_next_sp_with_pose
from ult.ult import Get_next_sp
from ult.apply_prior import apply_prior

import cv2
Expand Down Expand Up @@ -471,7 +471,7 @@ def generator1():
blobs['O_score'].append(Object[5])
blobs['H_boxes'].append(np.array(
[0, Human_out[2][0], Human_out[2][1], Human_out[2][2], Human_out[2][3]]))
blobs['sp'].append(Get_next_sp_with_pose(Human_out[2], Object[2], Human_out[6]))
blobs['sp'].append(Get_next_sp(Human_out[2], Object[2]))
mask = np.zeros(shape=(im_shape[0] // 16, im_shape[1] // 16, 1), dtype=np.float32)
obj_box = blobs['O_boxes'][-1][1:].astype(np.int32)
# print(obj_box)
Expand Down
3 changes: 3 additions & 0 deletions lib/models/train_Solver_HICO_MultiBatch.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,16 +93,19 @@ def construct_graph(self, sess):
self.gt_class_HO[
self.net.pos1_idx:self.net.get_compose_num_stop()]],
compose_type)
new_loss = new_loss['vcl_loss']
elif self.net.model_name.__contains__('atl'):
new_loss = self.compose_feature_helper.merge_generate(O_features,
V_features,
[self.gt_class_HO[:self.net.pos1_idx], self.gt_class_HO[self.net.pos1_idx:self.net.get_compose_num_stop()]],
'atl')
new_loss = new_loss['vcl_loss']
else:
new_loss = self.compose_feature_helper.merge_generate(O_features,
V_features,
[self.gt_class_HO[:self.net.pos1_idx], self.gt_class_HO[self.net.pos1_idx:self.net.get_compose_num_stop()]],
compose_type)
new_loss = new_loss['vcl_loss']
ll = self.compose_feature_helper.get_ll()
tower_losses.append(new_loss * ll)
if self.net.model_name.__contains__('pret') and self.net.model_name.__contains__('classifier'):
Expand Down
1 change: 1 addition & 0 deletions lib/models/train_Solver_HICO_MultiGPU.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ def construct_graph2(self, sess):
:num_stop_list[j]] for j in
range(2)],
compose_type)
new_loss = new_loss['vcl_loss']
ll = self.compose_feature_helper.get_ll()
tower_losses.append(new_loss * ll)

Expand Down
2 changes: 2 additions & 0 deletions lib/models/train_Solver_VCOCO_HOI.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,13 +106,15 @@ def construct_graph2(self, sess):
:num_stop_list[j]] for j in
range(2)],
'atl')
new_loss = new_loss['vcl_loss']
# atl1 means we jointly composing HOI images and object images
else:
new_loss = self.compose_feature_helper.merge_generate(O_features, V_features,
[self.blobs[j][key][
:num_stop_list[j]] for j in
range(2)],
compose_type)
new_loss = new_loss['vcl_loss']
ll = self.compose_feature_helper.get_ll()
tower_losses.append(new_loss * ll)
variables = tf.trainable_variables()
Expand Down
1 change: 1 addition & 0 deletions lib/models/train_Solver_VCOCO_MultiGPU.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@ def construct_graph2(self, sess):
:num_stop_list[j]] for j in
range(2)],
compose_type)
if type(new_loss) == dict: new_loss = new_loss['vcl_loss']
ll = self.compose_feature_helper.get_ll()
tower_losses.append(new_loss * ll)
variables = tf.trainable_variables()
Expand Down
45 changes: 43 additions & 2 deletions lib/networks/HOI.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,14 @@
elif os.environ['DATASET'] == 'HICO_res101':
from networks.ResNet101_HICO import ResNet101, resnet_arg_scope
parent_model = ResNet101
elif os.environ['DATASET'] == 'HICO_res101_icl':
from networks.ResNet101_HICO_zs import ResNet101, resnet_arg_scope

parent_model = ResNet101
elif os.environ['DATASET'] == 'HICO_icl':
from networks.ResNet50_HICO_zs import ResNet50, resnet_arg_scope

parent_model = ResNet50
elif os.environ['DATASET'] == 'VCOCO1':
from networks.ResNet50_VCOCO_HOI import ResNet50, resnet_arg_scope
parent_model = ResNet50
Expand All @@ -52,6 +60,39 @@ def __init__(self, model_name='VCL_union_multi_ml5_def1_l2_rew2_aug5_3_x5new_res
self.losses['fake_D_total_loss'] = 0
self.losses['fake_G_total_loss'] = 0
self.losses['fake_total_loss'] = 0
if self.model_name.__contains__('affordance'):
self.affordance_count = tf.get_variable("affordance_count", shape=[self.verb_num_classes, self.obj_num_classes],
initializer=tf.zeros_initializer)
if self.model_name.__contains__('OFFLINE'): # For ablation study, you can ignore this
if self.model_name.__contains__('VCOCO'):
init_v = np.load('/project/ZHIHOU//afford/iCAN_R_union_multi_ml5_l05_t5_VERB_def2_aug5_3_new_VCOCO_test_CL_21_affordance_9_HOI_iter_160000.ckpt.npy')
else:
if self.model_name.__contains__('inito'):
init_v = np.load('/project/ZHIHOU//afford/iCAN_R_union_batch_large2_ml5_def1_vloss2_VERB_l2_aug5_3_x5new_res101_affordance_inito_OFFLINE1_AF713_r_1_9ATL_2.npy')
elif self.model_name.__contains__('OFFLINE1'):
init_v = np.load('/project/ZHIHOU/afford/iCAN_R_union_batch_large2_ml5_def1_vloss2_VERB_l2_aug5_3_x5new_res101_affordance_9ATL_2.npy')
else:
init_v = np.load("/project/ZHIHOU//afford/iCAN_R_union_batch_large2_ml5_def1_vloss2_VERB_l2_aug5_3_x5new_res101_affordance_AF713_OFFLINE_9ATL_2.npy")
init_v = init_v.reshape([self.verb_num_classes, self.obj_num_classes])
self.affordance_stat = tf.get_variable("affordance_stat", shape=[self.verb_num_classes, self.obj_num_classes], dtype=tf.float32,
trainable=True,
initializer=tf.constant_initializer(init_v),
regularizer=regularizers.l2_regularizer(0.001))
elif self.model_name.__contains__('inito'): # For debug, you can ignore this
init_v = np.load('/project/ZHIHOU/afford/iCAN_R_union_batch_large2_ml5_def1_vloss2_VERB_l2_aug5_3_x5new_res101_affordance_9ATL_2.npy')
init_v = init_v.reshape([self.verb_num_classes, self.obj_num_classes])
self.affordance_stat = tf.get_variable("affordance_stat", shape=[self.verb_num_classes, self.obj_num_classes], dtype=tf.float32,
trainable=True,
initializer=tf.constant_initializer(init_v),
regularizer=regularizers.l2_regularizer(0.001))
elif self.model_name.__contains__('init'):
self.affordance_stat = tf.get_variable("affordance_stat", shape=[self.verb_num_classes, self.obj_num_classes], dtype=tf.float32,
trainable=True,
initializer=tf.constant_initializer(np.matmul(self.verb_to_HO_matrix_np, self.obj_to_HO_matrix_np.transpose())),
regularizer=regularizers.l2_regularizer(0.001))
else:
self.affordance_stat = tf.get_variable("affordance_stat", shape=[self.verb_num_classes, self.obj_num_classes], dtype=tf.float32, trainable=True,
initializer=tf.zeros_initializer, regularizer=regularizers.l2_regularizer(0.001))

def set_gt_class_HO_for_G_verbs(self, gt_class_HO_for_G_verbs):
self.gt_class_HO_for_G_verbs = gt_class_HO_for_G_verbs
Expand Down Expand Up @@ -82,7 +123,7 @@ def res5_ho(self, pool5_HO, is_training, name):
scope=self.scope)
return fc7_HO

def head_to_tail_ho(self, fc7_O, fc7_verbs, fc7_O_raw, fc7_verbs_raw, is_training, name):
def head_to_tail_ho(self, fc7_O, fc7_verbs, fc7_O_raw, fc7_verbs_raw, is_training, name, gt_verb_class=None, gt_obj_class=None):
if name == 'fc_HO':
nameprefix = '' # TODO should improve
else:
Expand Down Expand Up @@ -131,7 +172,7 @@ def region_classification_ho(self, fc7_verbs, is_training, initializer, name, na
# return None
with tf.variable_scope(name) as scope:
if self.model_name.__contains__('VERB'):
cls_score_verbs = slim.fully_connected(fc7_verbs, self.verb_num_classes,
cls_score_hoi = slim.fully_connected(fc7_verbs, self.verb_num_classes,
weights_initializer=initializer,
trainable=is_training,
reuse=tf.AUTO_REUSE,
Expand Down
11 changes: 5 additions & 6 deletions lib/networks/HOI_Concept_Discovery.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@
import tensorflow.contrib.slim as slim


from networks.HOI import DisentanglingNet
from networks.tools1 import get_convert_matrix
from networks.HOI import HOI
from networks.tools import get_convert_matrix


import numpy as np
Expand All @@ -19,7 +19,7 @@
from ult.ult import get_zero_shot_type, get_unseen_index


class HOIICLNet(DisentanglingNet):
class HOIICLNet(HOI):
def __init__(self, model_name='iCAN_ResNet50_HICO', task_id = 0, incremental_class_pairs = [[(), (), (), ]]):
super(HOIICLNet, self).__init__(model_name)
self.task_id = task_id
Expand Down Expand Up @@ -54,6 +54,7 @@ def __init__(self, model_name='iCAN_ResNet50_HICO', task_id = 0, incremental_cla
continue
zs_convert_matrix[i][zs_j] = 1
zs_j += 1
print(self.verb_num_classes)
verb_to_HO_matrix, obj_to_HO_matrix = get_convert_matrix(self.verb_num_classes, self.obj_num_classes)
new_v_hoi = np.matmul(verb_to_HO_matrix, zs_convert_matrix)
new_o_hoi = np.matmul(obj_to_HO_matrix, zs_convert_matrix)
Expand Down Expand Up @@ -222,9 +223,6 @@ def add_loss(self):
# verb loss
temp = self.add_verb_loss(num_stop)
loss += temp
# sim loss
temp = self.cal_sim_loss(num_stop)
loss += temp

# interactiveness
interactiveness_loss = 0
Expand All @@ -233,3 +231,4 @@ def add_loss(self):
print(self.losses)
print(self.predictions)
return loss

Loading

0 comments on commit 85e15d3

Please sign in to comment.