diff --git a/fastSum/PointerGen/README.md b/fastSum/PointerGen/README.md
index fcac06c6f92ab22c8fbfc0eea2ea3395f3101ec4..578f71da1f6f83fbdfcb0bb26ee17295dc0f5e3c 100644
--- a/fastSum/PointerGen/README.md
+++ b/fastSum/PointerGen/README.md
@@ -1,4 +1,29 @@
-# pointer-gen-fastnlp
+# Pointer Generator
-fastNLP实现的Get To The Point Summarization with Pointer-Generator Networks论文模型
-论文地址: https://arxiv.org/abs/1704.04368
+fastNLP实现的 [Get To The Point Summarization with Pointer-Generator Networks](https://arxiv.org/abs/1704.04368)论文模型
+
+原始代码 [地址](https://github.com/abisee/pointer-generator)
+
+## 包依赖
+- Python 3.7
+- [PyTorch](https://github.com/pytorch/pytorch) >= 1.1.0
+- [fastNLP](https://github.com/fastnlp/fastNLP) dev0.5.0
+- [pyrouge](https://github.com/bheinzerling/pyrouge) 0.1.3
+- [rouge](https://github.com/pltrdy/rouge) 1.0.0
+- [transformers](https://github.com/huggingface/transformers) 1.0.0
+
+## 输入数据格式
+这里要求输入文件每一行都是一个json dict，dict需要有两个必要的key：'text'和'summary'，分别代表原文档和对应摘要，其值要求是list，保存已经分句结束的结果。
+
+## 训练
+训练Pointer Generator，命令行如下：
+```
+python train.py -train_data_path TRAIN_DATA_PATH -eval_data_path VALID_DATA_PATH -log_root LOG_ROOT_NAME -is_pointer_gen -is_coverage -n_epochs 33 -visible_gpu 0 -lr_coverage 0.025 -batch_size 16
+```
+其中TRAIN_DATA_PATH，VALID_DATA_PATH，LOG_ROOT_NAME分别代表训练数据路径，dev集数据路径以及模型存储的文件夹路径。如果要去除pointer机制或者coverage机制，可以去掉is_pointer_gen和is_coverage。
+
+## 测试
+```
+python decode.py -decode_data_path TEST_DATA_PATH -train_data_path TRAIN_DATA_PATH -test_model CHECKPOINT -log_root LOG_ROOT_NAME -is_pointer_gen -is_coverage -test_data_name TEST_DATA_NAME -visible_gpu 0
+```
+其中TEST_DATA_PATH，TRAIN_DATA_PATH，CHECKPOINT，LOG_ROOT_NAME，TEST_DATA_NAME 分别代表测试数据路径，训练数据路径，测试模型路径，decode结果保存路径，测试数据集名字。其中is_pointer_gen和is_coverage的加入要和训练模型时一致。
diff --git a/fastSum/PointerGen/data_util/config.py b/fastSum/PointerGen/data_util/config.py
deleted file mode 100644
index 7d994742fe379a8b5f06a4220e1e7090aeebaa45..0000000000000000000000000000000000000000
--- a/fastSum/PointerGen/data_util/config.py
+++ /dev/null
@@ -1,46 +0,0 @@
-import os
-
-
-class Config():
- def __init__(self):
- super(Config, self).__init__()
- self.root_dir = "/remote-home/yrchen"
- #"tasks/fastnlp-relevant/summarization/cnn-dailymail/finished_files/chunked/train_*"
- self.train_data_path = "/remote-home/yrchen/Datasets/CNNDM/finished_files_new/CNNDM.train.json"
- #"tasks/fastnlp-relevant/summarization/cnn-dailymail/finished_files/val.bin"
- self.eval_data_path = "/remote-home/yrchen/Datasets/CNNDM/finished_files_new/CNNDM.val.json"
- #"tasks/fastnlp-relevant/summarization/cnn-dailymail/finished_files/test.bin"
- self.decode_data_path ="/remote-home/yrchen/Datasets/CNNDM/finished_files_new/CNNDM.test.json"
- self.vocab_path = os.path.join(self.root_dir,
- "tasks/fastnlp-relevant/summarization/cnn-dailymail/finished_files/vocab")
- self.log_root = os.path.join(self.root_dir, "tasks/fastnlp-relevant/summarization/my-pnt-sum/log/CNNDM")
- self.train_path = None
- self.model_path = None
-
- # Hyperparameters
- self.hidden_dim = 256
- self.emb_dim = 128
- self.batch_size = 32
- self.max_enc_steps = 400
- self.max_dec_steps = 100
- self.beam_size = 4
- self.min_dec_steps = 35
- #这个要随着不同数据集的变化而变化
- self.vocab_size = 50000
-
- self.lr = 0.15
- self.adagrad_init_acc = 0.1
- self.rand_unif_init_mag = 0.02
- self.trunc_norm_init_std = 1e-4
- self.max_grad_norm = 2.0 #2.0
-
- self.pointer_gen = True
- self.is_coverage = True
- self.cov_loss_wt = 1.0
-
- self.eps = 1e-12
- self.n_epochs = 100
-
- self.use_gpu = True
-
- self.lr_coverage = 0.010#0.15
diff --git a/fastSum/PointerGen/data_util/data.py b/fastSum/PointerGen/data_util/data.py
index 430c86d44c0eca06b5e93478b7671d1af4ccff4e..f33274f0aded3e08378009ea6a2d7c390bacf2f5 100644
--- a/fastSum/PointerGen/data_util/data.py
+++ b/fastSum/PointerGen/data_util/data.py
@@ -19,127 +19,8 @@ UNKNOWN_TOKEN = '[UNK]' # This has a vocab id, which is used to represent out-o
START_DECODING = '[START]' # This has a vocab id, which is used at the start of every decoder input sequence
STOP_DECODING = '[STOP]' # This has a vocab id, which is used at the end of untruncated target sequences
-# Note: none of , , [PAD], [UNK], [START], [STOP] should appear in the vocab file.
-
-'''
-class Cnn_dailymailLodaer(DataSetLoader):
- def __init__(self):
- super(Cnn_dailymailLodaer, self).__init__()
-
- def _load(self, data_path):
- def text_generator(example_generator):
- _count_1 = 0
- _count_2 = 0
- while True:
- try:
- e = example_generator.__next__() # e is a tf.Example
- except StopIteration:
- break
- try:
- article_text = e.features.feature['article'].bytes_list.value[
- 0] # the article text was saved under the key 'article' in the data files
- abstract_text = e.features.feature['abstract'].bytes_list.value[
- 0] # the abstract text was saved under the key 'abstract' in the data files
- except ValueError:
- logger.error('Failed to get article or abstract from example')
- continue
- if len(article_text) == 0: # See https://github.com/abisee/pointer-generator/issues/1
- _count_1 += 1
- logger.warning('Found an example with empty article text. Skipping it. Skipping number: %d'%_count_1)
- continue
- else:
- #_count_2 += 1
- #logger.info("getting example: %d"%_count_2)
- yield (article_text, abstract_text)
-
- def example_generator(data_path):
- filelist = glob.glob(data_path) # get the list of datafiles
- assert filelist, ('Error: Empty filelist at %s' % data_path) # check filelist isn't empty
- filelist = sorted(filelist)
-
- for f in filelist:
- reader = open(f, 'rb')
- while True:
- len_bytes = reader.read(8)
- if not len_bytes: break # finished reading this file
- str_len = struct.unpack('q', len_bytes)[0]
- example_str = struct.unpack('%ds' % str_len, reader.read(str_len))[0]
- yield example_pb2.Example.FromString(example_str)
- logger.info("example_generator completed reading all datafiles. No more data.")
-
-
- input_gen = text_generator(example_generator(data_path))
- data_dict = {"article": [], "abstract_sentences": []}
-
- while True:
- try:
- (article,
- abstract) = input_gen.__next__() # read the next example from file. article and abstract are both strings.
- if isinstance(abstract, bytes):
- abstract = str(abstract, encoding="utf-8")
- if isinstance(article, bytes):
- article = str(article, encoding="utf-8")
-
- except StopIteration: # if there are no more examples:
- logger.info("The example generator for this example queue filling thread has exhausted data.")
- break
-
- abstract_sentences = [sent.strip() for sent in abstract2sents(
- abstract)] # Use the and tags in abstract to get a list of sentences.
-
- data_dict["article"].append(article)
- data_dict["abstract_sentences"].append(abstract_sentences)
-
- dataset = DataSet(data_dict)
-
- return dataset
-
- def process(self, paths, vocab_path, vocab_size):
- def read_vocab(vocab_file, max_size):
- word_list = []
- count = 0
-
- for w in [UNKNOWN_TOKEN, PAD_TOKEN, START_DECODING, STOP_DECODING]:
- word_list.append(w)
- count += 1
-
- with open(vocab_file, 'r') as vocab_f:
- for line in vocab_f:
- pieces = line.split()
- if len(pieces) != 2:
- logger.warning('Warning: incorrectly formatted line in vocabulary file: %s\n' % line)
- continue
- w = pieces[0]
- if w in [SENTENCE_START, SENTENCE_END, UNKNOWN_TOKEN, PAD_TOKEN, START_DECODING, STOP_DECODING]:
- raise Exception(
- ', , [UNK], [PAD], [START] and [STOP] shouldn\'t be in the vocab file, but %s is' % w)
- if w in word_list:
- raise Exception('Duplicated word in vocabulary file: %s' % w)
- word_list.append(w)
- count += 1
- if max_size != 0 and count >= max_size:
- logger.info("max_size of vocab was specified as %i; we now have %i words. Stopping reading." % (
- max_size, count))
- break
-
- logger.info(
- "Finished constructing vocabulary of %i total words. Last word added: %s" % (count, word_list[-1]))
- return word_list
-
- vocab = Vocabulary(padding=PAD_TOKEN, unknown=UNKNOWN_TOKEN)
- vocab.update(read_vocab(vocab_path, vocab_size))
- datasets = {}
- for key, value in paths.items():
-
- logger.info("-"*5+"processing dataset " + key+"-"*5)
- datasets[key] = self._load(value)
- datasets[key].apply(lambda x: ' '.join(x["abstract_sentences"]), new_field_name='abstract')
- logger.info("dataset " + key + " size is %d" % len(datasets[key]))
- logger.info("-"*5+"process dataset "+key+" done!"+"-"*5)
-
- return DataBundle(vocabs={"train": vocab}, datasets=datasets)
-'''
+# Note: none of , , [PAD], [UNK], [START], [STOP] should appear in the vocab file.
def convert_list_to_ndarray(field):
diff --git a/fastSum/PointerGen/data_util/utils.py b/fastSum/PointerGen/data_util/utils.py
index 2aa61fa5b5c31955b7f8388469b97a1fc0a0f783..77b2fbb5ad53135756df2372e8c585cb905dfcf8 100644
--- a/fastSum/PointerGen/data_util/utils.py
+++ b/fastSum/PointerGen/data_util/utils.py
@@ -70,7 +70,8 @@ def clean(x):
def pyrouge_score_all(hyps_list, refer_list, config, remap=True):
nowTime = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
- PYROUGE_ROOT = os.path.join('/remote-home/yrchen/', nowTime)
+ PYROUGE_ROOT = os.path.join('~/tmp', nowTime)
+
SYSTEM_PATH = os.path.join(PYROUGE_ROOT, 'gold')
MODEL_PATH = os.path.join(PYROUGE_ROOT, 'system')
if os.path.exists(SYSTEM_PATH):
@@ -103,16 +104,14 @@ def pyrouge_score_all(hyps_list, refer_list, config, remap=True):
f.write(hyps.replace("\n", " "))
f.write("\n")
- # r = Rouge155('/remote-home/dqwang/ROUGE/RELEASE-1.5.5')
- # r = pyrouge.Rouge155()
- r = pyrouge.Rouge155('/remote-home/yrchen/ROUGE/ROUGE/RELEASE-1.5.5')
+ r = pyrouge.Rouge155('../../resources/ROUGE/RELEASE-1.5.5')
r.system_dir = SYSTEM_PATH
r.model_dir = MODEL_PATH
r.system_filename_pattern = 'Reference.(\d+).txt'
r.model_filename_pattern = 'Model.[A-Z].#ID#.txt'
- output = r.convert_and_evaluate(rouge_args="-e /remote-home/yrchen/ROUGE/ROUGE/RELEASE-1.5.5/data -a -m -n 2 -d")
+ output = r.convert_and_evaluate(rouge_args="-e ../../resources/ROUGE/RELEASE-1.5.5/data -a -m -n 2 -d")
# output = r.convert_and_evaluate()
output_dict = r.output_to_dict(output)
@@ -136,7 +135,7 @@ def pyrouge_score_all(hyps_list, refer_list, config, remap=True):
def pyrouge_score_all_multi(hyps_list, refer_list, config, remap=True):
nowTime = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
- PYROUGE_ROOT = os.path.join('/remote-home/yrchen/', nowTime)
+ PYROUGE_ROOT = os.path.join('~/tmp', nowTime)
SYSTEM_PATH = os.path.join(PYROUGE_ROOT, 'system')
MODEL_PATH = os.path.join(PYROUGE_ROOT, 'gold')
if os.path.exists(SYSTEM_PATH):
@@ -176,7 +175,6 @@ def pyrouge_score_all_multi(hyps_list, refer_list, config, remap=True):
f.write(hyps)
f.write("\n")
- # r = Rouge155('/remote-home/dqwang/ROUGE/RELEASE-1.5.5')
r = pyrouge.Rouge155()
r.system_dir = SYSTEM_PATH
@@ -184,7 +182,6 @@ def pyrouge_score_all_multi(hyps_list, refer_list, config, remap=True):
r.system_filename_pattern = 'Model.(\d+).txt'
r.model_filename_pattern = 'Reference.[A-Z].#ID#.txt'
- # output = r.convert_and_evaluate(rouge_args="-e /remote-home/dqwang/ROUGE/RELEASE-1.5.5/data -a -m -n 2 -d")
output = r.convert_and_evaluate()
output_dict = r.output_to_dict(output)
diff --git a/fastSum/PointerGen/decode.py b/fastSum/PointerGen/decode.py
index 206606449c6271ef28134db56ede19de204b4696..a10d304e4e266d5d6de03402f54aab224ba66c6a 100644
--- a/fastSum/PointerGen/decode.py
+++ b/fastSum/PointerGen/decode.py
@@ -51,10 +51,10 @@ def initial_dir(mode, config, model_file_path=None):
if not os.path.exists(decode_path):
os.mkdir(decode_path)
else:
- if os.path.exists(decode_path+"/"+"gold.txt"):
- os.remove(decode_path+"/"+"gold.txt")
- if os.path.exists(decode_path+"/"+"pred.txt"):
- os.remove(decode_path+"/"+"pred.txt")
+ if os.path.exists(decode_path + "/" + "gold.txt"):
+ os.remove(decode_path + "/" + "gold.txt")
+ if os.path.exists(decode_path + "/" + "pred.txt"):
+ os.remove(decode_path + "/" + "pred.txt")
return decode_path
@@ -82,12 +82,6 @@ def run_test(model_file_path, config):
state = torch.load(model_file_path, map_location=lambda storage, location: storage).state_dict()
model.load_state_dict(state)
- '''
- model.encoder.load_state_dict(state['encoder_state_dict'])
- model.decoder.load_state_dict(state['decoder_state_dict'], strict=False)
- model.reduce_state.load_state_dict(state['reduce_state_dict'])
- '''
-
tester = Tester(model=model, data=datainfo.datasets['test'], metrics=PyRougeMetric(pred='prediction',
art_oovs='article_oovs',
abstract_sentences='abstract_sentences',
@@ -110,8 +104,6 @@ def getting_k_model_path(path, top_k):
return [os.path.join(path, _item[0]) for _item in k_result]
-# python decode.py -decode_data_path CNNDM/finished_files_new1/CNNDM.test.json -train_data_path CNNDM/finished_files_new1/CNNDM.train.json -test_model ../log/CNNDM/train_1576560623/model/model_223000_1576669601 -log_root CNNDM -is_pointer_gen -is_coverage -test_data_name cnndm -visible_gpu 5
-# python decode.py -decode_data_path CNNDM/finished_files_new1/CNNDM.test.json -train_data_path CNNDM/finished_files_new1/CNNDM.train.json -m ../log/CNNDM/train_pointer_gen_coverage/model/ -log_root CNNDM -is_pointer_gen -is_coverage -test_data_name cnndm -visible_gpu 5 -top_k 5
if __name__ == '__main__':
parser = argparse.ArgumentParser(description="Train script")
parser.add_argument("-top_k", default=1, help="choose the k lowest loss model to test", type=int)
@@ -119,21 +111,14 @@ if __name__ == '__main__':
help="Model file for retraining (default: None).")
parser.add_argument('-visible_gpu', default=-1, type=int, required=True)
- parser.add_argument('-dataset_path', default="/remote-home/yrchen/Datasets")
parser.add_argument('-train_data_path',
- default="CNNDM/finished_files_new1/CNNDM.train.json", required=True)
- # parser.add_argument('-eval_data_path',
- # default="CNNDM/finished_files_new1/CNNDM.val.json", required=True)
+ default="", required=True, help="path of train data")
parser.add_argument('-decode_data_path',
- default="CNNDM/finished_files_new1/CNNDM.test.json", required=True)
- # parser.add_argument('-vocab_path', default='CNNDM/finished_files_new1/vocab.pkl')
- parser.add_argument('-root',
- default='/remote-home/yrchen/tasks/fastnlp-relevant/summarization/my-pnt-sum/log')
- parser.add_argument('-log_root', default='CNNDM', required=True)
+ default="", required=True, help="path of test data")
+ parser.add_argument('-log_root', default='', required=True, help="root to save result")
parser.add_argument('-hidden_dim', default=256, type=int)
parser.add_argument('-emb_dim', default=128, type=int)
- # parser.add_argument('-batch_size', default=8, type=int)
parser.add_argument('-batch_size', default=32, type=int)
parser.add_argument('-max_enc_steps', default=400, type=int)
parser.add_argument('-max_dec_steps', default=100, type=int)
@@ -161,13 +146,6 @@ if __name__ == '__main__':
parser.add_argument('-test_model', default='', type=str)
args = parser.parse_args()
- args.train_data_path = os.path.join(args.dataset_path, args.train_data_path)
- # args.eval_data_path = os.path.join(args.dataset_path, args.eval_data_path)
- args.decode_data_path = os.path.join(args.dataset_path, args.decode_data_path)
- # args.vocab_path = os.path.join(args.dataset_path, args.vocab_path)
-
- args.log_root = os.path.join(args.root, args.log_root)
-
if args.visible_gpu != -1:
args.use_gpu = True
torch.cuda.set_device(args.visible_gpu)
diff --git a/fastSum/PointerGen/train.py b/fastSum/PointerGen/train.py
index 1198abac86ce671963ba093a3fb15daf1404498c..bfb14a9ac609021fbc3a67a5e4ef8f5a8475cfc5 100644
--- a/fastSum/PointerGen/train.py
+++ b/fastSum/PointerGen/train.py
@@ -103,7 +103,6 @@ def run_train(config):
'reduce_state_dict': model.reduce_state.state_dict()
}
torch.save(state, bestmodel_save_path)
- # 不是作为形参传入到Trainer里面的么，怎么里面的model变化会影响到外面的？
logger.info('[INFO] Saving eval best model to %s', bestmodel_save_path)
@@ -112,21 +111,14 @@ if __name__ == '__main__':
parser.add_argument("-m", dest="model_file_path", required=False, default=None,
help="Model file for retraining (default: None).")
parser.add_argument('-visible_gpu', default=-1, type=int, required=True)
- parser.add_argument('-dataset_path', default="/remote-home/yrchen/Datasets")
parser.add_argument('-train_data_path',
- default="CNNDM/finished_files_new1/CNNDM.train.json", required=True)
+ default="", required=True, help="the path of training data")
parser.add_argument('-eval_data_path',
- default="CNNDM/finished_files_new1/CNNDM.val.json", required=True)
- # parser.add_argument('-decode_data_path',
- # default="CNNDM/finished_files_new1/CNNDM.test.json", required=True)
- # parser.add_argument('-vocab_path', default='CNNDM/finished_files_new1/vocab.pkl')
- parser.add_argument('-root',
- default='/remote-home/yrchen/tasks/fastnlp-relevant/summarization/my-pnt-sum/log')
- parser.add_argument('-log_root', default='CNNDM', required=True)
+ default="", required=True, help="the path of development data")
+ parser.add_argument('-log_root', default='', required=True, help="root to save result")
parser.add_argument('-hidden_dim', default=256, type=int)
parser.add_argument('-emb_dim', default=128, type=int)
- # parser.add_argument('-batch_size', default=8, type=int)
parser.add_argument('-batch_size', default=16, type=int)
parser.add_argument('-max_enc_steps', default=400, type=int)
parser.add_argument('-max_dec_steps', default=100, type=int)
@@ -146,19 +138,11 @@ if __name__ == '__main__':
parser.add_argument('-cov_loss_wt', default=1.0, type=float)
parser.add_argument('-eps', default=1e-12, type=float)
- # parser.add_argument('-max_iterations', default=500000, required=True, type=int)
parser.add_argument("-n_epochs", default=33, type=int, required=True)
parser.add_argument('-lr_coverage', default=0.15, type=float)
args = parser.parse_args()
- args.train_data_path = os.path.join(args.dataset_path, args.train_data_path)
- args.eval_data_path = os.path.join(args.dataset_path, args.eval_data_path)
- # args.decode_data_path = os.path.join(args.dataset_path, args.decode_data_path)
- # args.vocab_path = os.path.join(args.dataset_path, args.vocab_path)
-
- args.log_root = os.path.join(args.root, args.log_root)
-
if args.visible_gpu != -1:
args.use_gpu = True
torch.cuda.set_device(args.visible_gpu)
diff --git a/fastSum/PointerGen/training_ptr_gen/main.py b/fastSum/PointerGen/training_ptr_gen/main.py
deleted file mode 100644
index 9b35ecaebe41723673eb99e5f03be7c9ab499d97..0000000000000000000000000000000000000000
--- a/fastSum/PointerGen/training_ptr_gen/main.py
+++ /dev/null
@@ -1,126 +0,0 @@
-from data_util.config import Config
-from data_util.data import prepare_dataInfo, PAD_TOKEN
-from data_util.logging import logger
-from model.loss import MyLoss
-from model.model import Model
-from fastNLP import BucketSampler
-from fastNLP import Trainer
-from fastNLP import Tester
-from torch.optim import Adagrad
-from model.metric import PyRougeMetric, FastRougeMetric
-import os
-import time
-from data_util.utils import print_config, write_eval_results
-from training_ptr_gen.callback import TrainCallback
-import torch
-import sys
-import tensorflow as tf
-
-config = Config()
-
-
-def initial_dir(mode, model_file_path=None):
- if mode == 'train':
- train_dir = os.path.join(config.log_root, 'train_%d' % (int(time.time())))
- if not os.path.exists(train_dir):
- os.mkdir(train_dir)
-
- model_dir = os.path.join(train_dir, 'model')
- if not os.path.exists(model_dir):
- os.mkdir(model_dir)
- return train_dir, model_dir
-
- else:
- if model_file_path is None:
- logger.error("error!, no model to load")
- raise Exception("empty model file path!", model_file_path)
- parent_path = os.path.dirname(model_file_path)
- train_path = os.path.dirname(parent_path)
- model_name = os.path.basename(model_file_path)
- decode_path = os.path.join(train_path, 'decode_%s' % (model_name))
-
- if not os.path.exists(decode_path):
- os.mkdir(decode_path)
-
- return decode_path
-
-
-def set_up_data(mode):
- datainfo = prepare_dataInfo(mode, config.train_data_path, config.eval_data_path, config.decode_data_path,
- config.vocab_path, config.vocab_size, config)
- logger.info('-' * 10 + "set up data done!" + '-' * 10)
- return datainfo
-
-
-def run_train():
- train_dir, model_dir = initial_dir('train')
- config.train_path = train_dir
- config.model_path = model_dir
- print_config(config, train_dir)
- datainfo = set_up_data('train')
- train_sampler = BucketSampler(batch_size=config.batch_size, seq_len_field_name='enc_len')
- criterion = MyLoss(config=config, padding_idx=datainfo.vocabs["train"].to_index(PAD_TOKEN))
-
- model = Model(vocab=datainfo.vocabs["train"])
- params = list(model.encoder.parameters()) + list(model.decoder.parameters()) + \
- list(model.reduce_state.parameters())
- initial_lr = config.lr_coverage if config.is_coverage else config.lr
- optimizer = Adagrad(params, lr=initial_lr, initial_accumulator_value=config.adagrad_init_acc)
-
- train_loader = datainfo.datasets["train"]
- valid_loader = datainfo.datasets["dev"]
- summary_writer = tf.compat.v1.summary.FileWriter(train_dir)
- trainer = Trainer(model=model, train_data=train_loader, optimizer=optimizer, loss=criterion,
- batch_size=config.batch_size, check_code_level=-1,
- n_epochs=config.n_epochs, print_every=100, dev_data=valid_loader,
- metrics=FastRougeMetric(pred='prediction', art_oovs='article_oovs',
- abstract_sentences='abstract_sentences', config=config,
- vocab=datainfo.vocabs["train"]),
- metric_key="rouge-l-f", validate_every=-1, save_path=model_dir,
- callbacks=[TrainCallback(config, summary_writer, patience=5)], use_tqdm=False)
-
- logger.info("-" * 5 + "start training" + "-" * 5)
-
- traininfo = trainer.train(load_best_model=True)
- logger.info(' | end of Train | time: {:5.2f}s | '.format(traininfo["seconds"]))
- logger.info('[INFO] best eval model in epoch %d and iter %d', traininfo["best_epoch"], traininfo["best_step"])
- logger.info(traininfo["best_eval"])
-
- bestmodel_save_path = os.path.join(config.model_path,
- 'bestmodel.pkl') # this is where checkpoints of best models are saved
- state = {
- 'encoder_state_dict': model.encoder.state_dict(),
- 'decoder_state_dict': model.decoder.state_dict(),
- 'reduce_state_dict': model.reduce_state.state_dict()
- }
- torch.save(state, bestmodel_save_path)
- # 不是作为形参传入到Trainer里面的么，怎么里面的model变化会影响到外面的？
- logger.info('[INFO] Saving eval best model to %s', bestmodel_save_path)
-
-
-def run_test(model_file_path):
- decode_path = initial_dir('test', model_file_path)
- datainfo = set_up_data('test')
- model = Model(vocab=datainfo.vocabs["train"])
- tester = Tester(datainfo.datasets['test'], model=model, metrics=PyRougeMetric(pred='prediction',
- art_oovs='article_oovs',
- abstract_sentences='abstract_sentences',
- config=config,
- vocab=datainfo.vocabs["train"]), batch_size=1)
- eval_results = tester.test()
- write_eval_results(decode_path, eval_results)
-
-
-if __name__ == '__main__':
- torch.cuda.set_device(4)
- mode = sys.argv[1]
- if mode == 'train':
- logger.info("------start mode train------")
- run_train()
- elif mode == 'test':
- logger.info("------start mode test-------")
- model_filename = sys.argv[2]
- run_test(model_filename)
- else:
- logger.error("error: none of the mode is in train or test!")
- raise Exception("wrong mode! neither train nor test!", mode)
diff --git a/fastSum/PreSum/README.md b/fastSum/PreSum/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..a00ebf4c845bf017c46c66aed1812303e677b254
--- /dev/null
+++ b/fastSum/PreSum/README.md
@@ -0,0 +1,65 @@
+# PreSum
+FastNLP实现的EMNLP2019论文 [Text Summarization with Pretrained Encoders](https://arxiv.org/pdf/1908.08345)
+
+原始代码[地址](https://github.com/nlpyang/PreSumm)
+
+
+## 包依赖
+- Python 3.7
+- [PyTorch](https://github.com/pytorch/pytorch) >= 1.0.1
+- [fastNLP](https://github.com/fastnlp/fastNLP) 0.6.0
+- [pyrouge](https://github.com/bheinzerling/pyrouge) 0.1.3
+- [rouge](https://github.com/pltrdy/rouge) 1.0.0
+- [transformers](https://github.com/huggingface/transformers) >= 1.2.0
+
+
+
+## 数据预处理
+调用preprocess.py来预处理数据。示例如下：
+```
+python preprocess.py --raw_path INPUT_PATH --save_path data/OUTPUT_PATH --log_file LOG_PATH
+```
+其中INPUT_PATH，OUTPUT_PATH和LOG_PATH分别代表预处理的输入目录，输出目录名字以及log的路径。
+这里要求INPUT_PATH目录下有xx.train.jsonl， xx.val.jsonl， xx.test.jsonl 命名的文件，同时要求输入文件每一行都是一个json dict，dict需要有两个必要的key：'text'和'summary'，分别代表原文档和对应摘要，其值要求是list，保存已经分句结束的结果。
+
+
+
+## TransformerABS
+训练transformer为基础结构的生成式模型，命令行如下：
+```
+python train_presum.py -task abs -mode train -dec_dropout 0.2 -save_path SAVE_DIR -sep_optim true -lr_bert 0.002 -lr_dec 0.2 -save_checkpoint_steps 2000 -batch_size 16 -accum_count 5 -use_bert_emb true -warmup_steps_bert 20000 -warmup_steps_dec 10000 -max_pos 512 -visible_gpus 0,1,2,3 -log_file LOG_PATH -label_type INPUT_DIR -valid_steps 2000 -n_epochs 10 -max_summary_len 600
+```
+其中SAVE_DIR, LOG_PATH和INPUT_DIR分别代表了存储训练模型的目录，log的路径以及输入数据的文件夹名字。特别的，INPUT_DIR对应了数据预处理部分的OUTPUT_PATH，只需要告知data目录下输入的文件夹名字即可。
+
+
+测试transformer为基础结构的生成式模型，命令行如下：
+```
+python train_presum.py -task abs -mode test -test_batch_size 12 -log_file LOG_PATH -test_from CHECKPOINT_PATH -sep_optim true -visible_gpus 0 -max_pos 512 -max_length 200 -alpha 0.95 -min_length 50 -block_trigram True -label_type INPUT_DIR -max_summary_len 600 -decode_path DECODE_PATH
+```
+其中LOG_PATH，CHECKPOINT_PATH，INPUT_DIR，DECODE_PATH分别代表了log存储路径，测试模型的路径，包含测试集的文件夹名字（对应了数据预处理部分的OUTPUT_PATH），以及保存decode结果的路径名字。max_length ，min_length 和alpha 需要根据不同数据集特性调节。
+
+
+
+## BERTSUMABS
+训练BERT为基础结构的生成式模型，命令行如下：
+```
+python train_presum.py -task abs -mode train -dec_dropout 0.1 -save_path SAVE_DIR -sep_optim False -lr 0.05 -save_checkpoint_steps 2000 -batch_size 8 -accum_count 8 -use_bert_emb true -warmup_steps 10000 -max_pos 512 -visible_gpus 0,1,2,3 -log_file LOG_PATH -label_type INPUT_DIR -valid_steps 2000 -n_epochs 10 -max_summary_len 600 -encoder baseline -enc_dropout 0.1 -enc_hidden_size 512 -enc_layers 6 -enc_ff_size 2048 -dec_layers 6 -dec_hidden_size 512 -dec_ff_size 2048
+```
+其中SAVE_DIR, LOG_PATH和INPUT_DIR分别代表了存储训练模型的目录，log的路径以及输入数据的文件夹名字。特别的，INPUT_DIR对应了数据预处理部分的OUTPUT_PATH，只需要告知data目录下输入的文件夹名字即可。
+
+
+测试BERT为基础结构的生成式模型，命令行如下：
+```
+python train_presum.py -task abs -mode test -test_batch_size 12 -log_file LOG_PATH -test_from CHECKPOINT_PATH -sep_optim False -visible_gpus 0 -max_pos 512 -max_length 200 -alpha 0.95 -min_length 50 -block_trigram True -label_type INPUT_DIR -max_summary_len 300 -decode_path DECODE_PATH -encoder baseline
+```
+其中LOG_PATH，CHECKPOINT_PATH，INPUT_DIR，DECODE_PATH分别代表了log存储路径，测试模型的路径，包含测试集的文件夹名字（对应了数据预处理部分的OUTPUT_PATH），以及保存decode结果的路径名字。max_length ，min_length 和alpha 需要根据不同数据集特性调节。
+
+## TransformerEXT
+训练transformer为基础结构的抽取式模型，命令行如下：
+测试transformer为基础结构的抽取式模型，命令行如下：
+
+
+
+## BERTSUMEXT
+训练BERT为基础结构的抽取式模型，命令行如下：
+测试BERT为基础结构的抽取式模型，命令行如下：
diff --git a/fastSum/PreSum/__init__.py b/fastSum/PreSum/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/fastSum/PreSum/callback.py b/fastSum/PreSum/callback.py
new file mode 100644
index 0000000000000000000000000000000000000000..377f49d5e1767e19f3f1f79baa637ed161fc3df8
--- /dev/null
+++ b/fastSum/PreSum/callback.py
@@ -0,0 +1,281 @@
+import os
+import torch
+import sys
+from torch import nn
+
+from fastNLP.core.callback import Callback, EarlyStopError
+from others.utils import mkdir
+from fastNLP.core.utils import _get_model_device
+import fitlog
+from copy import deepcopy
+from fastNLP import Tester, DataSet
+
+
+class MyCallback(Callback):
+ def __init__(self, args, optims):
+ super(MyCallback, self).__init__()
+ self.args = args
+ self.real_step = 0
+ self.optims = optims
+
+ def on_step_end(self):
+ if self.step % self.update_every == 0 and self.step > 0:
+ self.real_step += 1
+ cur_lr = []
+ for o in self.optims:
+ cur_lr.append("{:.8f}".format(o.optimizer.param_groups[0]['lr']))
+ o.step()
+
+ if self.real_step % 1000 == 0:
+ self.pbar.write('Current learning rate is {}, real_step: {}'.format("|".join(cur_lr), self.real_step))
+
+ def on_epoch_end(self):
+ self.pbar.write('Epoch {} is done !!!'.format(self.epoch))
+
+
+def _save_model(checkpoint, model_name, save_dir, only_param=True):
+ """ 存储不含有显卡信息的 state_dict 或 model
+ 这里还没有实现保存optims, 相关实现可以继续
+ :param model:
+ :param model_name:
+ :param save_dir: 保存的 directory
+ :param only_param:
+ :return:
+ """
+
+ model_path = os.path.join(save_dir, model_name)
+
+ if not os.path.isdir(save_dir):
+ os.makedirs(save_dir, exist_ok=True)
+ if isinstance(checkpoint['model'], nn.DataParallel):
+ checkpoint['model'] = checkpoint['model'].module
+ if only_param:
+ state_dict = checkpoint['model'].state_dict()
+ for key in state_dict:
+ state_dict[key] = state_dict[key].cpu()
+ checkpoint['model'] = state_dict
+ torch.save(checkpoint, model_path)
+ else:
+ _model_device = _get_model_device(checkpoint['model'])
+ checkpoint['model'].cpu()
+ torch.save(checkpoint, model_path)
+ checkpoint['model'].to(_model_device)
+
+
+class SaveModelCallback(Callback):
+ """
+ 由于Trainer在训练过程中只会保存最佳的模型， 该 callback 可实现多种方式的结果存储。
+ 会根据训练开始的时间戳在 save_dir 下建立文件夹，在再文件夹下存放多个模型
+ -save_dir
+ -2019-07-03-15-06-36
+ -epoch0step20{metric_key}{evaluate_performance}.pt # metric是给定的metric_key, evaluate_perfomance是性能
+ -epoch1step40
+ -2019-07-03-15-10-00
+ -epoch:0step:20{metric_key}:{evaluate_performance}.pt # metric是给定的metric_key, evaluate_perfomance是性能
+ :param str save_dir: 将模型存放在哪个目录下，会在该目录下创建以时间戳命名的目录，并存放模型
+ :param int top: 保存dev表现top多少模型。-1为保存所有模型
+ :param bool only_param: 是否只保存模型权重
+ :param save_on_exception: 发生exception时，是否保存一份当时的模型
+ """
+
+ def __init__(self, save_dir, optims, args, top=5, only_param=False, save_on_exception=False):
+ super().__init__()
+
+ if not os.path.isdir(save_dir):
+ raise IsADirectoryError("{} is not a directory.".format(save_dir))
+ self.save_dir = save_dir
+ if top < 0:
+ self.top = sys.maxsize
+ else:
+ self.top = top
+ self.optims = optims
+ self._ordered_save_models = [] # List[Tuple], Tuple[0]是metric， Tuple[1]是path。metric是依次变好的，所以从头删
+
+ self.only_param = only_param
+ self.save_on_exception = save_on_exception
+ self.args = args
+
+ def on_train_begin(self):
+ self.save_dir = os.path.join(self.save_dir, self.trainer.start_time)
+ if not os.path.exists(self.save_dir):
+ os.mkdir(self.save_dir)
+
+ def on_valid_end(self, eval_result, metric_key, optimizer, is_better_eval):
+ metric_value = list(eval_result.values())[0][metric_key]
+ self._save_this_model(metric_value)
+
+ def _insert_into_ordered_save_models(self, pair):
+ # pair:(metric_value, model_name)
+ # 返回save的模型pair与删除的模型pair. pair中第一个元素是metric的值，第二个元素是模型的名称
+ index = -1
+ for _pair in self._ordered_save_models:
+ if _pair[0] >= pair[0] and self.trainer.increase_better:
+ break
+ if not self.trainer.increase_better and _pair[0] <= pair[0]:
+ break
+ index += 1
+ save_pair = None
+ if len(self._ordered_save_models) < self.top or (len(self._ordered_save_models) >= self.top and index != -1):
+ save_pair = pair
+ self._ordered_save_models.insert(index + 1, pair)
+ delete_pair = None
+ if len(self._ordered_save_models) > self.top:
+ delete_pair = self._ordered_save_models.pop(0)
+ return save_pair, delete_pair
+
+ def _save_this_model(self, metric_value):
+ name = "epoch:{}_step:{}_{}:{:.6f}.pt".format(self.epoch, self.step, self.trainer.metric_key, metric_value)
+ save_pair, delete_pair = self._insert_into_ordered_save_models((metric_value, name))
+ checkpoint = {'model': self.model, 'optims': self.optims, 'opt': self.args}
+ if save_pair:
+ try:
+ _save_model(checkpoint, model_name=name, save_dir=self.save_dir)
+ except Exception as e:
+ print(f"The following exception:{e} happens when saves model to {self.save_dir}.")
+ if delete_pair:
+ try:
+ delete_model_path = os.path.join(self.save_dir, delete_pair[1])
+ if os.path.exists(delete_model_path):
+ os.remove(delete_model_path)
+ except Exception as e:
+ print(f"Fail to delete model {name} at {self.save_dir} caused by exception:{e}.")
+
+ def on_exception(self, exception):
+ if self.save_on_exception:
+ checkpoint = {'model': self.model, 'optims': self.optims, 'opt': self.args}
+ name = "epoch:{}_step:{}_Exception:{}.pt".format(self.epoch, self.step, exception.__class__.__name__)
+ _save_model(checkpoint, model_name=name, save_dir=self.save_dir)
+
+
+class EarlyStopCallback(Callback):
+ r"""
+ 多少个epoch没有变好就停止训练，相关类 :class:`~fastNLP.core.callback.EarlyStopError`
+ """
+
+ def __init__(self, patience=10):
+ r"""
+
+ :param int patience: epoch的数量
+ """
+ super(EarlyStopCallback, self).__init__()
+ self.patience = patience
+ self.wait = 0
+
+ def on_valid_end(self, eval_result, metric_key, optimizer, is_better_eval):
+ if not is_better_eval:
+ # current result is getting worse
+ if self.wait == self.patience:
+ raise EarlyStopError("Early stopping raised.")
+ else:
+ self.wait += 1
+ else:
+ self.wait = 0
+
+ def on_exception(self, exception):
+ if isinstance(exception, EarlyStopError):
+ print("Early Stopping triggered in epoch {}!".format(self.epoch))
+ else:
+ raise exception # 抛出陌生Error
+
+
+class FitlogCallback(Callback):
+ r"""
+ 该callback可将loss和progress写入到fitlog中; 如果Trainer有dev的数据，将自动把dev的结果写入到log中; 同时还支持传入
+ 一个(或多个)test数据集进行测试(只有在trainer具有dev时才能使用)，每次在dev上evaluate之后会在这些数据集上验证一下。
+ 并将验证结果写入到fitlog中。这些数据集的结果是根据dev上最好的结果报道的，即如果dev在第3个epoch取得了最佳，则
+ fitlog中记录的关于这些数据集的结果就是来自第三个epoch的结果。
+ """
+
+ def __init__(self, data=None, tester=None, log_loss_every=0, verbose=1, log_exception=False):
+ r"""
+
+ :param ~fastNLP.DataSet,Dict[~fastNLP.DataSet] data: 传入DataSet对象，会使用多个Trainer中的metric对数据进行验证。如果需要
+ 传入多个DataSet请通过dict的方式传入，dict的key将作为对应dataset的name传递给fitlog。data的结果的名称以'data'开头。
+ :param ~fastNLP.Tester,Dict[~fastNLP.Tester] tester: Tester对象，将在on_valid_end时调用。tester的结果的名称以'tester'开头
+ :param int log_loss_every: 多少个step记录一次loss(记录的是这几个batch的loss平均值)，如果数据集较大建议将该值设置得
+ 大一些，不然会导致log文件巨大。默认为0, 即不要记录loss。
+ :param int verbose: 是否在终端打印evaluation的结果，0不打印。
+ :param bool log_exception: fitlog是否记录发生的exception信息
+ """
+ super().__init__()
+ self.datasets = {}
+ self.testers = {}
+ self._log_exception = log_exception
+ assert isinstance(log_loss_every, int) and log_loss_every >= 0
+ if tester is not None:
+ if isinstance(tester, dict):
+ for name, test in tester.items():
+ if not isinstance(test, Tester):
+ raise TypeError(f"{name} in tester is not a valid fastNLP.Tester.")
+ self.testers['tester-' + name] = test
+ if isinstance(tester, Tester):
+ self.testers['tester-test'] = tester
+ for tester in self.testers.values():
+ setattr(tester, 'verbose', 0)
+
+ if isinstance(data, dict):
+ for key, value in data.items():
+ assert isinstance(value, DataSet), f"Only DataSet object is allowed, not {type(value)}."
+ for key, value in data.items():
+ self.datasets['data-' + key] = value
+ elif isinstance(data, DataSet):
+ self.datasets['data-test'] = data
+ elif data is not None:
+ raise TypeError("data receives dict[DataSet] or DataSet object.")
+
+ self.verbose = verbose
+ self._log_loss_every = log_loss_every
+ self._avg_loss = 0
+
+ def on_train_begin(self):
+ if (len(self.datasets) > 0 or len(self.testers) > 0) and self.trainer.dev_data is None:
+ raise RuntimeError("Trainer has no dev data, you cannot pass extra data to do evaluation.")
+
+ if len(self.datasets) > 0:
+ for key, data in self.datasets.items():
+ tester = Tester(data=data, model=self.model,
+ batch_size=self.trainer.kwargs.get('dev_batch_size', self.batch_size),
+ metrics=self.trainer.metrics,
+ verbose=0,
+ use_tqdm=self.trainer.test_use_tqdm,
+ sampler=self.trainer.kwargs.get('test_sampler', None))
+ self.testers[key] = tester
+ fitlog.add_progress(total_steps=self.n_steps)
+
+ def on_backward_begin(self, loss):
+ if self._log_loss_every > 0:
+ self._avg_loss += loss.item()
+ if self.step % self._log_loss_every == 0:
+ fitlog.add_loss(self._avg_loss / self._log_loss_every * self.update_every, name='loss',
+ step=int(self.step / self.update_every),
+ epoch=self.epoch)
+ self._avg_loss = 0
+
+ def on_valid_end(self, eval_result, metric_key, optimizer, better_result):
+ if better_result:
+ eval_result = deepcopy(eval_result)
+ eval_result['step'] = self.step
+ eval_result['epoch'] = self.epoch
+ fitlog.add_best_metric(eval_result)
+ fitlog.add_metric(eval_result, step=self.step, epoch=self.epoch)
+ if len(self.testers) > 0:
+ for key, tester in self.testers.items():
+ try:
+ eval_result = tester.test()
+ if self.verbose != 0:
+ self.pbar.write("FitlogCallback evaluation on {}:".format(key))
+ self.pbar.write(tester._format_eval_results(eval_result))
+ fitlog.add_metric(eval_result, name=key, step=self.step, epoch=self.epoch)
+ if better_result:
+ fitlog.add_best_metric(eval_result, name=key)
+ except Exception as e:
+ self.pbar.write("Exception happens when evaluate on DataSet named `{}`.".format(key))
+ raise e
+
+ def on_train_end(self):
+ fitlog.finish()
+
+ def on_exception(self, exception):
+ fitlog.finish(status=1)
+ if self._log_exception:
+ fitlog.add_other(repr(exception), name='except_info')
diff --git a/fastSum/PreSum/dataloader.py b/fastSum/PreSum/dataloader.py
new file mode 100644
index 0000000000000000000000000000000000000000..f999c65f7a6e2a97f3e32330cb3918e6fe51c915
--- /dev/null
+++ b/fastSum/PreSum/dataloader.py
@@ -0,0 +1,262 @@
+import bisect
+from time import time
+from datetime import timedelta
+
+from fastNLP.io.loader import JsonLoader
+from fastNLP.modules.tokenizer import BertTokenizer
+from fastNLP.io.data_bundle import DataBundle
+from fastNLP.core.const import Const
+
+
+class BertData(JsonLoader):
+
+ def __init__(self, max_nsents=60, max_ntokens=100, max_len=512):
+
+ fields = {'article': 'article',
+ 'label': 'label'}
+ super(BertData, self).__init__(fields=fields)
+
+ self.max_nsents = max_nsents
+ self.max_ntokens = max_ntokens
+ self.max_len = max_len
+
+ self.tokenizer = BertTokenizer.from_pretrained('/path/to/uncased_L-12_H-768_A-12')
+ self.cls_id = self.tokenizer.vocab['[CLS]']
+ self.sep_id = self.tokenizer.vocab['[SEP]']
+ self.pad_id = self.tokenizer.vocab['[PAD]']
+
+ def _load(self, paths):
+ dataset = super(BertData, self)._load(paths)
+ return dataset
+
+ def process(self, paths):
+
+ def truncate_articles(instance, max_nsents=self.max_nsents, max_ntokens=self.max_ntokens):
+ article = [' '.join(sent.lower().split()[:max_ntokens]) for sent in instance['article']]
+ return article[:max_nsents]
+
+ def truncate_labels(instance):
+ label = list(filter(lambda x: x < len(instance['article']), instance['label']))
+ return label
+
+ def bert_tokenize(instance, tokenizer, max_len, pad_value):
+ article = instance['article']
+ article = ' [SEP] [CLS] '.join(article)
+ word_pieces = tokenizer.tokenize(article)[:(max_len - 2)]
+ word_pieces = ['[CLS]'] + word_pieces + ['[SEP]']
+ token_ids = tokenizer.convert_tokens_to_ids(word_pieces)
+ while len(token_ids) < max_len:
+ token_ids.append(pad_value)
+ assert len(token_ids) == max_len
+ return token_ids
+
+ def get_seg_id(instance, max_len, sep_id):
+ _segs = [-1] + [i for i, idx in enumerate(instance['article']) if idx == sep_id]
+ segs = [_segs[i] - _segs[i - 1] for i in range(1, len(_segs))]
+ segment_id = []
+ for i, length in enumerate(segs):
+ if i % 2 == 0:
+ segment_id += length * [0]
+ else:
+ segment_id += length * [1]
+ while len(segment_id) < max_len:
+ segment_id.append(0)
+ return segment_id
+
+ def get_cls_id(instance, cls_id):
+ classification_id = [i for i, idx in enumerate(instance['article']) if idx == cls_id]
+ return classification_id
+
+ def get_labels(instance):
+ labels = [0] * len(instance['cls_id'])
+ label_idx = list(filter(lambda x: x < len(instance['cls_id']), instance['label']))
+ for idx in label_idx:
+ labels[idx] = 1
+ return labels
+
+ datasets = {}
+ for name in paths:
+ datasets[name] = self._load(paths[name])
+
+ # remove empty samples
+ datasets[name].drop(lambda ins: len(ins['article']) == 0 or len(ins['label']) == 0)
+
+ # truncate articles
+ datasets[name].apply(lambda ins: truncate_articles(ins, self.max_nsents, self.max_ntokens),
+ new_field_name='article')
+
+ # truncate labels
+ datasets[name].apply(truncate_labels, new_field_name='label')
+
+ # tokenize and convert tokens to id
+ datasets[name].apply(lambda ins: bert_tokenize(ins, self.tokenizer, self.max_len, self.pad_id),
+ new_field_name='article')
+
+ # get segment id
+ datasets[name].apply(lambda ins: get_seg_id(ins, self.max_len, self.sep_id), new_field_name='segment_id')
+
+ # get classification id
+ datasets[name].apply(lambda ins: get_cls_id(ins, self.cls_id), new_field_name='cls_id')
+
+ # get label
+ datasets[name].apply(get_labels, new_field_name='label')
+
+ # rename filed
+ datasets[name].rename_field('article', Const.INPUTS(0))
+ datasets[name].rename_field('segment_id', Const.INPUTS(1))
+ datasets[name].rename_field('cls_id', Const.INPUTS(2))
+ datasets[name].rename_field('lbael', Const.TARGET)
+
+ # set input and target
+ datasets[name].set_input(Const.INPUTS(0), Const.INPUTS(1), Const.INPUTS(2))
+ datasets[name].set_target(Const.TARGET)
+
+ # set paddding value
+ datasets[name].set_pad_val('article', 0)
+
+ return DataBundle(datasets=datasets)
+
+
+class PreSummABSLoader(JsonLoader):
+ """
+
+ """
+
+ def __init__(self, args):
+ fields = {
+ 'text_ids': 'text_ids',
+ 'summary_ids': Const.TARGET,
+ 'segment_ids': 'segment_ids',
+ 'cls_ids': 'cls_ids',
+ 'label': 'label',
+ 'summary': 'summary'
+ }
+ self.max_pos = args.max_pos
+ self.max_summary_len = args.max_summary_len
+
+ super(PreSummABSLoader, self).__init__(fields=fields)
+
+ def _load(self, paths):
+ dataset = super(PreSummABSLoader, self)._load(paths)
+ return dataset
+
+ def process(self, paths):
+ def truncate_input(instance):
+ text_ids = instance['text_ids']
+ summary_ids = instance[Const.TARGET]
+ cls_ids = instance['cls_ids']
+ segment_ids = instance['segment_ids']
+ label = instance['label']
+ tgt_txt = instance['summary']
+
+ end_id = [text_ids[-1]]
+ text_ids = text_ids[:-1][:self.max_pos - 1] + end_id
+ summary_ids = summary_ids[:self.max_summary_len][:-1] + [2]
+ segment_ids = segment_ids[:self.max_pos]
+ max_sent_id = bisect.bisect_left(cls_ids, self.max_pos)
+ label = label[:max_sent_id]
+ cls_ids = cls_ids[:max_sent_id]
+
+ return {'text_ids': text_ids, Const.TARGET: summary_ids, "summary_ids": summary_ids, 'cls_ids': cls_ids,
+ 'segment_ids': segment_ids, 'label': label, 'tgt_txt': tgt_txt}
+
+ print('Start loading datasets !!!')
+ start = time()
+
+ # load datasets
+ datasets = {}
+ for name in paths:
+ datasets[name] = self._load(paths[name])
+ print(name)
+ print(datasets[name][0])
+
+ datasets[name].apply_more(lambda ins: truncate_input(ins))
+
+ # set input and target
+ datasets[name].set_input('text_ids', 'segment_ids', 'cls_ids', 'tgt_txt', 'summary_ids')
+ datasets[name].set_target(Const.TARGET, 'tgt_txt')
+
+ # set padding value
+ # 如果使用其他的预训练模型要替换这里的pad value
+ datasets[name].set_pad_val('text_ids', 0)
+ datasets[name].set_pad_val('segment_ids', 0)
+ datasets[name].set_pad_val('summary_ids', 0)
+ datasets[name].set_pad_val('cls_ids', -1)
+ datasets[name].set_pad_val(Const.TARGET, 0)
+
+ print('Finished in {}'.format(timedelta(seconds=time() - start)))
+
+ return DataBundle(datasets=datasets)
+
+
+class PreSummEXTLoader(JsonLoader):
+
+ def __init__(self, args):
+ fields = {'text_ids': 'text_ids',
+ 'segment_ids': 'segment_ids',
+ 'cls_ids': 'cls_ids',
+ 'label': Const.TARGET,
+ 'summary': 'summary',
+ 'text': 'text',
+ }
+
+ self.max_pos = args.max_pos
+ self.max_summary_len = args.max_summary_len
+
+ super(PreSummEXTLoader, self).__init__(fields=fields)
+
+ def _load(self, paths):
+ dataset = super(PreSummEXTLoader, self)._load(paths)
+ return dataset
+
+ def process(self, paths):
+ def get_seq_len(instance):
+ return len(instance['text_ids'])
+
+ def truncate_input(instance):
+ text_ids = instance['text_ids']
+ cls_ids = instance['cls_ids']
+ segment_ids = instance['segment_ids']
+ label = instance[Const.TARGET]
+ tgt_txt = instance['summary']
+ src_txt = instance['text']
+
+ end_id = [text_ids[-1]]
+ text_ids = text_ids[:-1][:self.max_pos - 1] + end_id
+ segment_ids = segment_ids[:self.max_pos]
+ max_sent_id = bisect.bisect_left(cls_ids, self.max_pos)
+ label = label[:max_sent_id]
+ cls_ids = cls_ids[:max_sent_id]
+ assert len(label) == len(
+ cls_ids), "label and cls_ids size not match! Label size: {} while cls_ids size: {}".format(len(label),
+ len(cls_ids))
+
+ return {'text_ids': text_ids, 'cls_ids': cls_ids, 'src_txt': src_txt,
+ 'segment_ids': segment_ids, Const.TARGET: label, 'tgt_txt': tgt_txt}
+
+ print('Start loading datasets !!!')
+ start = time()
+
+ # load datasets
+ datasets = {}
+ for name in paths:
+ datasets[name] = self._load(paths[name])
+ print(name)
+ print(datasets[name][0])
+
+ datasets[name].apply_more(lambda ins: truncate_input(ins))
+ datasets[name].apply(get_seq_len, new_field_name='seq_len')
+
+ # set input and target
+ datasets[name].set_input('text_ids', 'segment_ids', 'cls_ids')
+ datasets[name].set_target(Const.TARGET, 'tgt_txt', 'src_txt')
+
+ # set padding value
+ datasets[name].set_pad_val('text_ids', 0)
+ datasets[name].set_pad_val('segment_ids', 0)
+ datasets[name].set_pad_val('cls_ids', -1)
+ datasets[name].set_pad_val(Const.TARGET, 0)
+
+ print('Finished in {}'.format(timedelta(seconds=time() - start)))
+
+ return DataBundle(datasets=datasets)
diff --git a/fastSum/PreSum/metrics.py b/fastSum/PreSum/metrics.py
new file mode 100644
index 0000000000000000000000000000000000000000..52fff2107ffcdba6e5613882f387017492ab7aa1
--- /dev/null
+++ b/fastSum/PreSum/metrics.py
@@ -0,0 +1,795 @@
+import numpy as np
+import json
+from os.path import join
+import torch
+import logging
+import tempfile
+import subprocess as sp
+from datetime import timedelta
+from time import time
+
+from pyrouge import Rouge155
+from pyrouge.utils import log
+
+from fastNLP.core.losses import LossBase
+from fastNLP.core.metrics import MetricBase
+
+import torch.nn.functional as F
+import torchsnooper
+ROOT = "."
+_ROUGE_PATH = '../resources/ROUGE/RELEASE-1.5.5'
+
+
+
+class LabelSmoothingLoss(torch.nn.Module):
+ """
+ With label smoothing,
+ KL-divergence between q_{smoothed ground truth prob.}(w)
+ and p_{prob. computed by model}(w) is minimized.
+ """
+
+ def __init__(self, label_smoothing, tgt_vocab_size, ignore_index=-100):
+ assert 0.0 < label_smoothing <= 1.0
+ self.padding_idx = ignore_index
+ super(LabelSmoothingLoss, self).__init__()
+
+ smoothing_value = label_smoothing / (tgt_vocab_size - 2)
+ one_hot = torch.full((tgt_vocab_size,), smoothing_value)
+ one_hot[self.padding_idx] = 0
+ self.register_buffer('one_hot', one_hot.unsqueeze(0))
+ self.confidence = 1.0 - label_smoothing
+
+ # @torchsnooper.snoop()
+ def forward(self, output, target):
+ """
+ output (FloatTensor): batch_size x n_classes
+ target (LongTensor): batch_size
+ """
+ # 我的修改：在原始代码基础上增加.to(torch.device(target.get_device())),注意这里要迁移到target在的特定device上，否则计算会出错
+ # model_prob = self.one_hot.repeat(target.size(0), 1)
+
+ model_prob = self.one_hot.repeat(target.size(0), 1).to(torch.device(target.get_device()))
+ model_prob.scatter_(1, target.unsqueeze(1), self.confidence)
+ model_prob.masked_fill_((target == self.padding_idx).unsqueeze(1), 0)
+ # print("LabelSmoothingLoss")
+ # print(output.shape, model_prob.shape)
+ return F.kl_div(output, model_prob, reduction='sum')
+
+
+class MyNLLLoss(LossBase):
+ def __init__(self, generator, vocab_size, pred=None, target=None, label_smoothing=0.0, pad_id=0):
+ super(MyNLLLoss, self).__init__()
+ self._init_param_map(pred=pred, target=target)
+ self.padding_idx = pad_id
+ self.generator = generator
+ # self.loss_func = torch.nn.BCELoss(reduction='none')
+ if label_smoothing > 0:
+ self.loss_func = LabelSmoothingLoss(
+ label_smoothing, vocab_size, ignore_index=self.padding_idx
+ )
+ else:
+ self.loss_func = torch.nn.NLLLoss(
+ ignore_index=self.padding_idx, reduction='sum'
+ )
+
+ def _bottle(self, _v):
+ return _v.view(-1, _v.size(2))
+
+ def get_loss(self, pred, target):
+ # print("MyNLLLoss")
+ # print(pred.shape, target.shape)
+ bottled_output = self._bottle(pred)
+ scores = self.generator(bottled_output)
+ # print(target.size())
+ target = target[:, 1:]
+ gtruth = target.contiguous().view(-1)
+
+ loss = self.loss_func(scores, gtruth)
+ # 不知道原来的这个mask是干什么用的 ?至少presum代码中没有用到mask
+ # loss = (loss * mask.float()).sum()
+ return loss
+
+
+class ABSLossMetric(MetricBase):
+ def __init__(self, generator, vocab_size, pred=None, target=None, label_smoothing=0.0, pad_id=0):
+ super(ABSLossMetric, self).__init__()
+ self._init_param_map(pred=pred, target=target)
+ self.padding_idx = pad_id
+ self.generator = generator
+ # self.loss_func = torch.nn.BCELoss(reduction='none')
+ if label_smoothing > 0:
+ self.loss_func = LabelSmoothingLoss(
+ label_smoothing, vocab_size, ignore_index=self.padding_idx
+ )
+ else:
+ self.loss_func = torch.nn.NLLLoss(
+ ignore_index=self.padding_idx, reduction='sum'
+ )
+ self.avg_loss = 0.0
+ self.nsamples = 0
+
+ def _bottle(self, _v):
+ return _v.view(-1, _v.size(2))
+
+ def evaluate(self, pred, target):
+ bottled_output = self._bottle(pred)
+ scores = self.generator(bottled_output)
+ target = target[:, 1:]
+ gtruth = target.contiguous().view(-1)
+ loss = self.loss_func(scores, gtruth)
+
+ batch_size = pred.size(0)
+
+ self.avg_loss += loss.item()
+ self.nsamples += batch_size
+
+ def get_metric(self, reset=True):
+ self.avg_loss = self.avg_loss / self.nsamples
+ eval_result = {'loss': self.avg_loss}
+ if reset:
+ self.avg_loss = 0
+ self.nsamples = 0
+ return eval_result
+
+
+class MyBCELoss(LossBase):
+
+ def __init__(self, pred=None, target=None, mask=None):
+ super(MyBCELoss, self).__init__()
+ self._init_param_map(pred=pred, target=target, mask=mask)
+ self.loss_func = torch.nn.BCELoss(reduction='none')
+
+ def get_loss(self, pred, target, mask):
+ loss = self.loss_func(pred, target.float())
+ loss = (loss * mask.float()).sum()
+ return loss
+
+
+class EXTLossMetric(MetricBase):
+ def __init__(self, pred=None, target=None, mask=None):
+ super(EXTLossMetric, self).__init__()
+ self._init_param_map(pred=pred, target=target, mask=mask)
+ self.loss_func = torch.nn.BCELoss(reduction='none')
+ self.avg_loss = 0.0
+ self.nsamples = 0
+
+ def evaluate(self, pred, target, mask):
+ batch_size = pred.size(0)
+ loss = self.loss_func(pred, target.float())
+ loss = (loss * mask.float()).sum()
+ self.avg_loss += loss
+ self.nsamples += batch_size
+
+ def get_metric(self, reset=True):
+ self.avg_loss = self.avg_loss / self.nsamples
+ eval_result = {'loss': self.avg_loss}
+ if reset:
+ self.avg_loss = 0
+ self.nsamples = 0
+ return eval_result
+
+
+import os
+import datetime
+from rouge import Rouge
+from others.utils import pyrouge_score_all, pyrouge_score_all_multi, ranstr
+
+
+def remend_score(scores_all):
+ remend_score = {}
+ for key, value in scores_all.items():
+ for subkey, subvalue in value.items():
+ remend_score[key + "-" + subkey] = subvalue
+ return remend_score
+
+
+def make_html_safe(s):
+ s = s.replace("<", "<")
+ s = s.replace(">", ">")
+ if "<" in s or ">" in s:
+ print("-------html not safe sent:")
+ print(s)
+ return s
+
+
+class RougeMetricEXT(MetricBase):
+ def __init__(self, n_ext=3, ngram_block=3, pred=None, src_txt=None, tgt_txt=None, mask=None, logger=None,
+ config=None):
+ super(RougeMetricEXT, self).__init__()
+ self._init_param_map(pred=pred, src_txt=src_txt, tgt_txt=tgt_txt, mask=mask)
+
+ self.n_ext = n_ext
+ self.ngram_block = ngram_block
+
+ self.referece = []
+ self.prediction = []
+
+ self.logger = logger
+ self.config = config
+
+ # Set model in validating mode.
+ def _get_ngrams(self, n, text):
+ ngram_set = set()
+ text_length = len(text)
+ max_index_ngram_start = text_length - n
+ for i in range(max_index_ngram_start + 1):
+ ngram_set.add(tuple(text[i:i + n]))
+ return ngram_set
+
+ def _block_tri(self, c, p):
+ tri_c = self._get_ngrams(self.ngram_block, c.split())
+ for s in p:
+ tri_s = self._get_ngrams(self.ngram_block, s.split())
+ if len(tri_c.intersection(tri_s)) > 0:
+ return True
+ return False
+
+ def evaluate(self, pred, src_txt, tgt_txt, mask):
+ sent_scores = pred + mask.float()
+ sent_scores = sent_scores.cpu().data.numpy()
+ selected_ids = np.argsort(-sent_scores, 1)
+ # selected_ids = np.sort(selected_ids,1)
+
+ for i, idx in enumerate(selected_ids):
+ _pred = []
+ if (len(src_txt[i]) == 0):
+ continue
+ for j in selected_ids[i][:len(src_txt[i])]:
+ if (j >= len(src_txt[i])):
+ continue
+ candidate = src_txt[i][j].strip()
+ if (self.ngram_block):
+ if (not self._block_tri(candidate, _pred)):
+ _pred.append(candidate)
+ else:
+ _pred.append(candidate)
+
+ if len(_pred) == self.n_ext:
+ break
+
+ _pred = '\n'.join([make_html_safe(sent) for sent in _pred])
+ self.prediction.append(_pred)
+
+ gold_sents = []
+ for sent in tgt_txt[i].split(''):
+ if len(sent.strip()) > 0:
+ gold_sents.append(sent.strip())
+ self.referece.append("\n".join(gold_sents))
+
+ def get_metric(self, reset=True):
+ pass
+
+
+class RougeMetricABS(MetricBase):
+ def __init__(self, pred=None, tgt_txt=None, config=None, vocab=None, logger=None):
+ super().__init__()
+
+ self.vocab = vocab
+ self.config = config
+ self._init_param_map(pred=pred, tgt_txt=tgt_txt)
+
+ self.prediction = []
+ self.referece = []
+
+ self.logger = logger
+
+ def evaluate(self, pred, tgt_txt):
+ """
+
+ :param prediction: [batch, N]
+ :param text: [batch, N]
+ :param summary: [batch, N]
+ :return:
+ """
+
+ batch_size = len(pred)
+
+ for b in range(batch_size):
+ # print(b,"----------------------",pred[b])
+ # output_ids = [int(id) for id in pred[b]]
+
+ pred_str = self.vocab.convert_ids_to_tokens([int(n) for n in pred[b][0]])
+ pred_str = ' '.join(pred_str).replace(' ##', '').replace('[unused0]', '').replace('[unused3]', '').replace(
+ '[PAD]', '').replace('[unused1]', '').replace(r' +', ' ').replace(' [unused2] ', '').replace(
+ '[unused2]', '').strip()
+ gold_str = tgt_txt[b]
+
+ pred_sents = []
+ for sent in pred_str.split(''):
+ if len(sent.strip()) > 0:
+ pred_sents.append(sent.strip())
+
+ abstract_sentences = []
+ for sent in gold_str.split(''):
+ if len(sent.strip()) > 0:
+ abstract_sentences.append(sent.strip())
+
+ self.prediction.append("\n".join([make_html_safe(sent) for sent in pred_sents]))
+ self.referece.append("\n".join([make_html_safe(sent) for sent in abstract_sentences]))
+
+ def get_metric(self, reset=True):
+ pass
+
+
+class FastRougeMetricABS(RougeMetricABS):
+ def __init__(self, pred=None, tgt_txt=None, config=None, vocab=None, logger=None):
+ super().__init__(pred, tgt_txt, config, vocab, logger)
+
+ def get_metric(self, reset=True):
+ self.logger.info("[INFO] Hyps and Refer number is %d, %d", len(self.prediction), len(self.referece))
+ if len(self.prediction) == 0 or len(self.referece) == 0:
+ self.logger.error("During testing, no hyps or refers is selected!")
+ return
+ rouge = Rouge()
+ scores_all = rouge.get_scores(self.prediction, self.referece, avg=True)
+ if reset:
+ self.prediction = []
+ self.referece = []
+ self.logger.info(scores_all)
+ scores_all = remend_score(scores_all)
+ return scores_all
+
+
+class PyRougeMetricABS(RougeMetricABS):
+ def __init__(self, pred=None, tgt_txt=None, config=None, vocab=None, logger=None):
+ super().__init__(pred, tgt_txt, config, vocab, logger)
+ nowTime = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
+ self.tmp_path = os.path.join(os.path.join(ROOT,'tmp'), "tmp_" + nowTime + "_" + ranstr(6))
+
+ def get_metric(self, reset=True):
+ self.logger.info("[INFO] Hyps and Refer number is %d, %d", len(self.prediction), len(self.referece))
+ if len(self.prediction) == 0 or len(self.referece) == 0:
+ self.logger.error("During testing, no hyps or refers is selected!")
+ return
+ if isinstance(self.referece[0], list):
+ self.logger.info("Multi Reference summaries!")
+ scores_all = pyrouge_score_all_multi(self.prediction, self.referece, self.config, self.tmp_path)
+ else:
+ scores_all = pyrouge_score_all(self.prediction, self.referece, self.config, self.tmp_path)
+ if reset:
+ self.prediction = []
+ self.referece = []
+ self.logger.info(scores_all)
+ return scores_all
+
+
+class FastRougeMetricEXT(RougeMetricEXT):
+ def __init__(self, n_ext=3, ngram_block=3, pred=None, src_txt=None, tgt_txt=None, mask=None, logger=None,
+ config=None):
+ super().__init__(n_ext, ngram_block, pred, src_txt, tgt_txt, mask, logger, config)
+
+ def get_metric(self, reset=True):
+ self.logger.info("[INFO] Hyps and Refer number is %d, %d", len(self.prediction), len(self.referece))
+ if len(self.prediction) == 0 or len(self.referece) == 0:
+ self.logger.error("During testing, no hyps or refers is selected!")
+ return
+ rouge = Rouge()
+ scores_all = rouge.get_scores(self.prediction, self.referece, avg=True)
+ if reset:
+ self.prediction = []
+ self.referece = []
+ self.logger.info(scores_all)
+ scores_all = remend_score(scores_all)
+ return scores_all
+
+
+class PyRougeMetricEXT(RougeMetricEXT):
+ def __init__(self, n_ext=3, ngram_block=3, pred=None, src_txt=None, tgt_txt=None, mask=None, logger=None,
+ config=None):
+ super().__init__(n_ext, ngram_block, pred, src_txt, tgt_txt, mask, logger, config)
+ nowTime = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
+ self.tmp_path = os.path.join(os.path.join(ROOT,'tmp'), "tmp_" + nowTime + "_" + ranstr(6))
+
+ def get_metric(self, reset=True):
+ self.logger.info("[INFO] Hyps and Refer number is %d, %d", len(self.prediction), len(self.referece))
+ if len(self.prediction) == 0 or len(self.referece) == 0:
+ self.logger.error("During testing, no hyps or refers is selected!")
+ return
+ if isinstance(self.referece[0], list):
+ self.logger.info("Multi Reference summaries!")
+ scores_all = pyrouge_score_all_multi(self.prediction, self.referece, self.config, self.tmp_path)
+ else:
+ scores_all = pyrouge_score_all(self.prediction, self.referece, self.config, self.tmp_path)
+ if reset:
+ self.prediction = []
+ self.referece = []
+ self.logger.info(scores_all)
+ return scores_all
+
+import numpy as np
+import json
+from os.path import join
+import torch
+import logging
+import tempfile
+import subprocess as sp
+from datetime import timedelta
+from time import time
+
+from pyrouge import Rouge155
+from pyrouge.utils import log
+
+from fastNLP.core.losses import LossBase
+from fastNLP.core.metrics import MetricBase
+
+import torch.nn.functional as F
+import torchsnooper
+
+_ROUGE_PATH = '/path/to/RELEASE-1.5.5'
+ROOT = "root/path"
+
+
+class LabelSmoothingLoss(torch.nn.Module):
+ """
+ With label smoothing,
+ KL-divergence between q_{smoothed ground truth prob.}(w)
+ and p_{prob. computed by model}(w) is minimized.
+ """
+
+ def __init__(self, label_smoothing, tgt_vocab_size, ignore_index=-100):
+ assert 0.0 < label_smoothing <= 1.0
+ self.padding_idx = ignore_index
+ super(LabelSmoothingLoss, self).__init__()
+
+ smoothing_value = label_smoothing / (tgt_vocab_size - 2)
+ one_hot = torch.full((tgt_vocab_size,), smoothing_value)
+ one_hot[self.padding_idx] = 0
+ self.register_buffer('one_hot', one_hot.unsqueeze(0))
+ self.confidence = 1.0 - label_smoothing
+
+ # @torchsnooper.snoop()
+ def forward(self, output, target):
+ """
+ output (FloatTensor): batch_size x n_classes
+ target (LongTensor): batch_size
+ """
+ # 我的修改：在原始代码基础上增加.to(torch.device(target.get_device())),注意这里要迁移到target在的特定device上，否则计算会出错
+ # model_prob = self.one_hot.repeat(target.size(0), 1)
+
+ model_prob = self.one_hot.repeat(target.size(0), 1).to(torch.device(target.get_device()))
+ model_prob.scatter_(1, target.unsqueeze(1), self.confidence)
+ model_prob.masked_fill_((target == self.padding_idx).unsqueeze(1), 0)
+ # print("LabelSmoothingLoss")
+ # print(output.shape, model_prob.shape)
+ return F.kl_div(output, model_prob, reduction='sum')
+
+
+class MyNLLLoss(LossBase):
+ def __init__(self, generator, vocab_size, pred=None, target=None, label_smoothing=0.0, pad_id=0):
+ super(MyNLLLoss, self).__init__()
+ self._init_param_map(pred=pred, target=target)
+ self.padding_idx = pad_id
+ self.generator = generator
+ # self.loss_func = torch.nn.BCELoss(reduction='none')
+ if label_smoothing > 0:
+ self.loss_func = LabelSmoothingLoss(
+ label_smoothing, vocab_size, ignore_index=self.padding_idx
+ )
+ else:
+ self.loss_func = torch.nn.NLLLoss(
+ ignore_index=self.padding_idx, reduction='sum'
+ )
+
+ def _bottle(self, _v):
+ return _v.view(-1, _v.size(2))
+
+ def get_loss(self, pred, target):
+ # print("MyNLLLoss")
+ # print(pred.shape, target.shape)
+ bottled_output = self._bottle(pred)
+ scores = self.generator(bottled_output)
+ # print(target.size())
+ target = target[:, 1:]
+ gtruth = target.contiguous().view(-1)
+
+ loss = self.loss_func(scores, gtruth)
+ # 不知道原来的这个mask是干什么用的 ?至少presum代码中没有用到mask
+ # loss = (loss * mask.float()).sum()
+ return loss
+
+
+class ABSLossMetric(MetricBase):
+ def __init__(self, generator, vocab_size, pred=None, target=None, label_smoothing=0.0, pad_id=0):
+ super(ABSLossMetric, self).__init__()
+ self._init_param_map(pred=pred, target=target)
+ self.padding_idx = pad_id
+ self.generator = generator
+ # self.loss_func = torch.nn.BCELoss(reduction='none')
+ if label_smoothing > 0:
+ self.loss_func = LabelSmoothingLoss(
+ label_smoothing, vocab_size, ignore_index=self.padding_idx
+ )
+ else:
+ self.loss_func = torch.nn.NLLLoss(
+ ignore_index=self.padding_idx, reduction='sum'
+ )
+ self.avg_loss = 0.0
+ self.nsamples = 0
+
+ def _bottle(self, _v):
+ return _v.view(-1, _v.size(2))
+
+ def evaluate(self, pred, target):
+ bottled_output = self._bottle(pred)
+ scores = self.generator(bottled_output)
+ target = target[:, 1:]
+ gtruth = target.contiguous().view(-1)
+ loss = self.loss_func(scores, gtruth)
+
+ batch_size = pred.size(0)
+
+ self.avg_loss += loss.item()
+ self.nsamples += batch_size
+
+ def get_metric(self, reset=True):
+ self.avg_loss = self.avg_loss / self.nsamples
+ eval_result = {'loss': self.avg_loss}
+ if reset:
+ self.avg_loss = 0
+ self.nsamples = 0
+ return eval_result
+
+
+class MyBCELoss(LossBase):
+
+ def __init__(self, pred=None, target=None, mask=None):
+ super(MyBCELoss, self).__init__()
+ self._init_param_map(pred=pred, target=target, mask=mask)
+ self.loss_func = torch.nn.BCELoss(reduction='none')
+
+ def get_loss(self, pred, target, mask):
+ loss = self.loss_func(pred, target.float())
+ loss = (loss * mask.float()).sum()
+ return loss
+
+
+class EXTLossMetric(MetricBase):
+ def __init__(self, pred=None, target=None, mask=None):
+ super(EXTLossMetric, self).__init__()
+ self._init_param_map(pred=pred, target=target, mask=mask)
+ self.loss_func = torch.nn.BCELoss(reduction='none')
+ self.avg_loss = 0.0
+ self.nsamples = 0
+
+ def evaluate(self, pred, target, mask):
+ batch_size = pred.size(0)
+ loss = self.loss_func(pred, target.float())
+ loss = (loss * mask.float()).sum()
+ self.avg_loss += loss
+ self.nsamples += batch_size
+
+ def get_metric(self, reset=True):
+ self.avg_loss = self.avg_loss / self.nsamples
+ eval_result = {'loss': self.avg_loss}
+ if reset:
+ self.avg_loss = 0
+ self.nsamples = 0
+ return eval_result
+
+
+import os
+import datetime
+from rouge import Rouge
+from others.utils import pyrouge_score_all, pyrouge_score_all_multi, ranstr
+
+
+def remend_score(scores_all):
+ remend_score = {}
+ for key, value in scores_all.items():
+ for subkey, subvalue in value.items():
+ remend_score[key + "-" + subkey] = subvalue
+ return remend_score
+
+
+def make_html_safe(s):
+ s = s.replace("<", "<")
+ s = s.replace(">", ">")
+ if "<" in s or ">" in s:
+ print("-------html not safe sent:")
+ print(s)
+ return s
+
+
+class RougeMetricEXT(MetricBase):
+ def __init__(self, n_ext=3, ngram_block=3, pred=None, src_txt=None, tgt_txt=None, mask=None, logger=None,
+ config=None):
+ super(RougeMetricEXT, self).__init__()
+ self._init_param_map(pred=pred, src_txt=src_txt, tgt_txt=tgt_txt, mask=mask)
+
+ self.n_ext = n_ext
+ self.ngram_block = ngram_block
+
+ self.referece = []
+ self.prediction = []
+
+ self.logger = logger
+ self.config = config
+
+ # Set model in validating mode.
+ def _get_ngrams(self, n, text):
+ ngram_set = set()
+ text_length = len(text)
+ max_index_ngram_start = text_length - n
+ for i in range(max_index_ngram_start + 1):
+ ngram_set.add(tuple(text[i:i + n]))
+ return ngram_set
+
+ def _block_tri(self, c, p):
+ tri_c = self._get_ngrams(self.ngram_block, c.split())
+ for s in p:
+ tri_s = self._get_ngrams(self.ngram_block, s.split())
+ if len(tri_c.intersection(tri_s)) > 0:
+ return True
+ return False
+
+ def evaluate(self, pred, src_txt, tgt_txt, mask):
+ sent_scores = pred + mask.float()
+ sent_scores = sent_scores.cpu().data.numpy()
+ selected_ids = np.argsort(-sent_scores, 1)
+ # selected_ids = np.sort(selected_ids,1)
+
+ for i, idx in enumerate(selected_ids):
+ _pred = []
+ if (len(src_txt[i]) == 0):
+ continue
+ for j in selected_ids[i][:len(src_txt[i])]:
+ if (j >= len(src_txt[i])):
+ continue
+ candidate = src_txt[i][j].strip()
+ if (self.ngram_block):
+ if (not self._block_tri(candidate, _pred)):
+ _pred.append(candidate)
+ else:
+ _pred.append(candidate)
+
+ if len(_pred) == self.n_ext:
+ break
+
+ _pred = '\n'.join([make_html_safe(sent) for sent in _pred])
+ self.prediction.append(_pred)
+
+ gold_sents = []
+ for sent in tgt_txt[i].split(''):
+ if len(sent.strip()) > 0:
+ gold_sents.append(sent.strip())
+ self.referece.append("\n".join(gold_sents))
+
+ def get_metric(self, reset=True):
+ pass
+
+
+class RougeMetricABS(MetricBase):
+ def __init__(self, pred=None, tgt_txt=None, config=None, vocab=None, logger=None):
+ super().__init__()
+
+ self.vocab = vocab
+ self.config = config
+ self._init_param_map(pred=pred, tgt_txt=tgt_txt)
+
+ self.prediction = []
+ self.referece = []
+
+ self.logger = logger
+
+ def evaluate(self, pred, tgt_txt):
+ """
+
+ :param prediction: [batch, N]
+ :param text: [batch, N]
+ :param summary: [batch, N]
+ :return:
+ """
+
+ batch_size = len(pred)
+
+ for b in range(batch_size):
+ # print(b,"----------------------",pred[b])
+ # output_ids = [int(id) for id in pred[b]]
+
+ pred_str = self.vocab.convert_ids_to_tokens([int(n) for n in pred[b][0]])
+ pred_str = ' '.join(pred_str).replace(' ##', '').replace('[unused0]', '').replace('[unused3]', '').replace(
+ '[PAD]', '').replace('[unused1]', '').replace(r' +', ' ').replace(' [unused2] ', '').replace(
+ '[unused2]', '').strip()
+ gold_str = tgt_txt[b]
+
+ pred_sents = []
+ for sent in pred_str.split(''):
+ if len(sent.strip()) > 0:
+ pred_sents.append(sent.strip())
+
+ abstract_sentences = []
+ for sent in gold_str.split(''):
+ if len(sent.strip()) > 0:
+ abstract_sentences.append(sent.strip())
+
+ self.prediction.append("\n".join([make_html_safe(sent) for sent in pred_sents]))
+ self.referece.append("\n".join([make_html_safe(sent) for sent in abstract_sentences]))
+
+ def get_metric(self, reset=True):
+ pass
+
+
+class FastRougeMetricABS(RougeMetricABS):
+ def __init__(self, pred=None, tgt_txt=None, config=None, vocab=None, logger=None):
+ super().__init__(pred, tgt_txt, config, vocab, logger)
+
+ def get_metric(self, reset=True):
+ self.logger.info("[INFO] Hyps and Refer number is %d, %d", len(self.prediction), len(self.referece))
+ if len(self.prediction) == 0 or len(self.referece) == 0:
+ self.logger.error("During testing, no hyps or refers is selected!")
+ return
+ rouge = Rouge()
+ scores_all = rouge.get_scores(self.prediction, self.referece, avg=True)
+ if reset:
+ self.prediction = []
+ self.referece = []
+ self.logger.info(scores_all)
+ scores_all = remend_score(scores_all)
+ return scores_all
+
+
+class PyRougeMetricABS(RougeMetricABS):
+ def __init__(self, pred=None, tgt_txt=None, config=None, vocab=None, logger=None):
+ super().__init__(pred, tgt_txt, config, vocab, logger)
+ nowTime = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
+ self.tmp_path = os.path.join(os.path.join(ROOT,'tmp'), "tmp_" + nowTime + "_" + ranstr(6))
+
+ def get_metric(self, reset=True):
+ self.logger.info("[INFO] Hyps and Refer number is %d, %d", len(self.prediction), len(self.referece))
+ if len(self.prediction) == 0 or len(self.referece) == 0:
+ self.logger.error("During testing, no hyps or refers is selected!")
+ return
+ if isinstance(self.referece[0], list):
+ self.logger.info("Multi Reference summaries!")
+ scores_all = pyrouge_score_all_multi(self.prediction, self.referece, self.config, self.tmp_path)
+ else:
+ scores_all = pyrouge_score_all(self.prediction, self.referece, self.config, self.tmp_path)
+ if reset:
+ self.prediction = []
+ self.referece = []
+ self.logger.info(scores_all)
+ return scores_all
+
+
+class FastRougeMetricEXT(RougeMetricEXT):
+ def __init__(self, n_ext=3, ngram_block=3, pred=None, src_txt=None, tgt_txt=None, mask=None, logger=None,
+ config=None):
+ super().__init__(n_ext, ngram_block, pred, src_txt, tgt_txt, mask, logger, config)
+
+ def get_metric(self, reset=True):
+ self.logger.info("[INFO] Hyps and Refer number is %d, %d", len(self.prediction), len(self.referece))
+ if len(self.prediction) == 0 or len(self.referece) == 0:
+ self.logger.error("During testing, no hyps or refers is selected!")
+ return
+ rouge = Rouge()
+ scores_all = rouge.get_scores(self.prediction, self.referece, avg=True)
+ if reset:
+ self.prediction = []
+ self.referece = []
+ self.logger.info(scores_all)
+ scores_all = remend_score(scores_all)
+ return scores_all
+
+
+class PyRougeMetricEXT(RougeMetricEXT):
+ def __init__(self, n_ext=3, ngram_block=3, pred=None, src_txt=None, tgt_txt=None, mask=None, logger=None,
+ config=None):
+ super().__init__(n_ext, ngram_block, pred, src_txt, tgt_txt, mask, logger, config)
+ nowTime = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
+ self.tmp_path = os.path.join(os.path.join(ROOT,'tmp'), "tmp_" + nowTime + "_" + ranstr(6))
+
+ def get_metric(self, reset=True):
+ self.logger.info("[INFO] Hyps and Refer number is %d, %d", len(self.prediction), len(self.referece))
+ if len(self.prediction) == 0 or len(self.referece) == 0:
+ self.logger.error("During testing, no hyps or refers is selected!")
+ return
+ if isinstance(self.referece[0], list):
+ self.logger.info("Multi Reference summaries!")
+ scores_all = pyrouge_score_all_multi(self.prediction, self.referece, self.config, self.tmp_path)
+ else:
+ scores_all = pyrouge_score_all(self.prediction, self.referece, self.config, self.tmp_path)
+ if reset:
+ self.prediction = []
+ self.referece = []
+ self.logger.info(scores_all)
+ return scores_all
diff --git a/fastSum/PreSum/models/__init__.py b/fastSum/PreSum/models/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/fastSum/PreSum/models/decoder.py b/fastSum/PreSum/models/decoder.py
new file mode 100644
index 0000000000000000000000000000000000000000..03e93bf433383b8fdfd41be7e76c39a72cf460c7
--- /dev/null
+++ b/fastSum/PreSum/models/decoder.py
@@ -0,0 +1,627 @@
+<<<<<<< HEAD
+"""
+Implementation of "Attention is All You Need"
+"""
+
+import torch
+import torch.nn as nn
+import numpy as np
+
+from models.encoder import PositionalEncoding
+from models.neural import MultiHeadedAttention, PositionwiseFeedForward, DecoderState
+
+MAX_SIZE = 5000
+
+
+class TransformerDecoderLayer(nn.Module):
+ """
+ Args:
+ d_model (int): the dimension of keys/values/queries in
+ MultiHeadedAttention, also the input size of
+ the first-layer of the PositionwiseFeedForward.
+ heads (int): the number of heads for MultiHeadedAttention.
+ d_ff (int): the second-layer of the PositionwiseFeedForward.
+ dropout (float): dropout probability(0-1.0).
+ self_attn_type (string): type of self-attention scaled-dot, average
+ """
+
+ def __init__(self, d_model, heads, d_ff, dropout):
+ super(TransformerDecoderLayer, self).__init__()
+
+ self.self_attn = MultiHeadedAttention(
+ heads, d_model, dropout=dropout)
+
+ self.context_attn = MultiHeadedAttention(
+ heads, d_model, dropout=dropout)
+ self.feed_forward = PositionwiseFeedForward(d_model, d_ff, dropout)
+ self.layer_norm_1 = nn.LayerNorm(d_model, eps=1e-6)
+ self.layer_norm_2 = nn.LayerNorm(d_model, eps=1e-6)
+ self.drop = nn.Dropout(dropout)
+ mask = self._get_attn_subsequent_mask(MAX_SIZE)
+ # Register self.mask as a buffer in TransformerDecoderLayer, so
+ # it gets TransformerDecoderLayer's cuda behavior automatically.
+ self.register_buffer('mask', mask)
+
+ def forward(self, inputs, memory_bank, src_pad_mask, tgt_pad_mask,
+ previous_input=None, layer_cache=None, step=None):
+ """
+ Args:
+ inputs (`FloatTensor`): `[batch_size x 1 x model_dim]`
+ memory_bank (`FloatTensor`): `[batch_size x src_len x model_dim]`
+ src_pad_mask (`LongTensor`): `[batch_size x 1 x src_len]`
+ tgt_pad_mask (`LongTensor`): `[batch_size x 1 x 1]`
+
+ Returns:
+ (`FloatTensor`, `FloatTensor`, `FloatTensor`):
+
+ * output `[batch_size x 1 x model_dim]`
+ * attn `[batch_size x 1 x src_len]`
+ * all_input `[batch_size x current_step x model_dim]`
+
+ """
+
+ dec_mask = torch.gt(tgt_pad_mask +
+ self.mask[:, :tgt_pad_mask.size(1),
+ :tgt_pad_mask.size(1)], 0)
+ # print("TransformerDecoderLayer.forward: ")
+ # print(inputs.size())
+ input_norm = self.layer_norm_1(inputs)
+ all_input = input_norm
+ if previous_input is not None:
+ # print(previous_input.size(), input_norm.size())
+ all_input = torch.cat((previous_input, input_norm), dim=1)
+ dec_mask = None
+
+ query = self.self_attn(all_input, all_input, input_norm,
+ mask=dec_mask,
+ layer_cache=layer_cache,
+ type="self")
+
+ query = self.drop(query) + inputs
+
+ query_norm = self.layer_norm_2(query)
+ mid = self.context_attn(memory_bank, memory_bank, query_norm,
+ mask=src_pad_mask,
+ layer_cache=layer_cache,
+ type="context")
+
+ output = self.feed_forward(self.drop(mid) + query)
+ return output, all_input
+ # return output
+
+ def _get_attn_subsequent_mask(self, size):
+ """
+ Get an attention mask to avoid using the subsequent info.
+
+ Args:
+ size: int
+
+ Returns:
+ (`LongTensor`):
+
+ * subsequent_mask `[1 x size x size]`
+ """
+ attn_shape = (1, size, size)
+ subsequent_mask = np.triu(np.ones(attn_shape), k=1).astype('uint8')
+ subsequent_mask = torch.from_numpy(subsequent_mask)
+ return subsequent_mask
+
+
+class TransformerDecoder(nn.Module):
+ """
+ The Transformer decoder from "Attention is All You Need".
+
+
+ .. mermaid::
+
+ graph BT
+ A[input]
+ B[multi-head self-attn]
+ BB[multi-head src-attn]
+ C[feed forward]
+ O[output]
+ A --> B
+ B --> BB
+ BB --> C
+ C --> O
+
+
+ Args:
+ num_layers (int): number of encoder layers.
+ d_model (int): size of the model
+ heads (int): number of heads
+ d_ff (int): size of the inner FF layer
+ dropout (float): dropout parameters
+ embeddings (:obj:`onmt.modules.Embeddings`):
+ embeddings to use, should have positional encodings
+ attn_type (str): if using a seperate copy attention
+ """
+
+ def __init__(self, num_layers, d_model, heads, d_ff, dropout, embeddings):
+ super(TransformerDecoder, self).__init__()
+
+ # Basic attributes.
+ self.decoder_type = 'transformer'
+ self.num_layers = num_layers
+ self.embeddings = embeddings
+ self.pos_emb = PositionalEncoding(dropout, self.embeddings.embedding_dim)
+
+ # Build TransformerDecoder.
+ self.transformer_layers = nn.ModuleList(
+ [TransformerDecoderLayer(d_model, heads, d_ff, dropout)
+ for _ in range(num_layers)])
+
+ self.layer_norm = nn.LayerNorm(d_model, eps=1e-6)
+
+ def forward(self, tgt, memory_bank, state, memory_lengths=None,
+ step=None, cache=None, memory_masks=None):
+ """
+ See :obj:`onmt.modules.RNNDecoderBase.forward()`
+ """
+
+ src_words = state.src
+ tgt_words = tgt
+ # print("tgt size: ", tgt.size())
+ src_batch, src_len = src_words.size()
+ tgt_batch, tgt_len = tgt_words.size()
+
+ # Run the forward pass of the TransformerDecoder.
+ # emb = self.embeddings(tgt, step=step)
+ emb = self.embeddings(tgt)
+ assert emb.dim() == 3 # len x batch x embedding_dim
+
+ output = self.pos_emb(emb, step)
+ # print("output before for roop: ", output.size())
+
+ src_memory_bank = memory_bank
+ padding_idx = self.embeddings.padding_idx
+ tgt_pad_mask = tgt_words.data.eq(padding_idx).unsqueeze(1) \
+ .expand(tgt_batch, tgt_len, tgt_len)
+
+ if (not memory_masks is None):
+ src_len = memory_masks.size(-1)
+ src_pad_mask = memory_masks.expand(src_batch, tgt_len, src_len)
+
+ else:
+ src_pad_mask = src_words.data.eq(padding_idx).unsqueeze(1) \
+ .expand(src_batch, tgt_len, src_len)
+
+ if state.cache is None:
+ saved_inputs = []
+ # print("state.cache:", state.cache)
+ for i in range(self.num_layers):
+ prev_layer_input = None
+ if state.cache is None:
+ if state.previous_input is not None:
+ # print("state.previous_input: ", state.previous_input.size())
+ # print("state.previous_layer_inputs[i]:", state.previous_layer_inputs[i].size())
+ prev_layer_input = state.previous_layer_inputs[i]
+ output, all_input \
+ = self.transformer_layers[i](
+ output, src_memory_bank,
+ src_pad_mask, tgt_pad_mask,
+ previous_input=prev_layer_input,
+ layer_cache=state.cache["layer_{}".format(i)]
+ if state.cache is not None else None,
+ step=step)
+ # if prev_layer_input is None:
+ # print("prev_layer_input is None")
+ # else:
+ # print("output size: {} previous_input_size: {}".format(output.size(), prev_layer_input.size()))
+ if state.cache is None:
+ saved_inputs.append(all_input)
+
+ if state.cache is None:
+ saved_inputs = torch.stack(saved_inputs)
+
+ output = self.layer_norm(output)
+
+ # Process the result and update the attentions.
+
+ if state.cache is None:
+ state = state.update_state(tgt, saved_inputs)
+
+ return output, state
+
+ def init_decoder_state(self, src, memory_bank,
+ with_cache=False):
+ """ Init decoder state """
+ state = TransformerDecoderState(src)
+ if with_cache:
+ state._init_cache(memory_bank, self.num_layers)
+ return state
+
+
+class TransformerDecoderState(DecoderState):
+ """ Transformer Decoder state base class """
+
+ def __init__(self, src):
+ """
+ Args:
+ src (FloatTensor): a sequence of source words tensors
+ with optional feature tensors, of size (len x batch).
+ """
+ self.src = src
+ self.previous_input = None
+ self.previous_layer_inputs = None
+ self.cache = None
+
+ @property
+ def _all(self):
+ """
+ Contains attributes that need to be updated in self.beam_update().
+ """
+ if (self.previous_input is not None
+ and self.previous_layer_inputs is not None):
+ return (self.previous_input,
+ self.previous_layer_inputs,
+ self.src)
+ else:
+ return (self.src,)
+
+ def detach(self):
+ if self.previous_input is not None:
+ self.previous_input = self.previous_input.detach()
+ if self.previous_layer_inputs is not None:
+ self.previous_layer_inputs = self.previous_layer_inputs.detach()
+ self.src = self.src.detach()
+
+ def update_state(self, new_input, previous_layer_inputs):
+ state = TransformerDecoderState(self.src)
+ state.previous_input = new_input
+ state.previous_layer_inputs = previous_layer_inputs
+ return state
+
+ def _init_cache(self, memory_bank, num_layers):
+ self.cache = {}
+
+ for l in range(num_layers):
+ layer_cache = {
+ "memory_keys": None,
+ "memory_values": None
+ }
+ layer_cache["self_keys"] = None
+ layer_cache["self_values"] = None
+ self.cache["layer_{}".format(l)] = layer_cache
+
+ def repeat_beam_size_times(self, beam_size):
+ """ Repeat beam_size times along batch dimension. """
+ self.src = self.src.data.repeat(1, beam_size, 1)
+
+ def map_batch_fn(self, fn):
+ def _recursive_map(struct, batch_dim=0):
+ for k, v in struct.items():
+ if v is not None:
+ if isinstance(v, dict):
+ _recursive_map(v)
+ else:
+ struct[k] = fn(v, batch_dim)
+
+ # print("self.src.size()", self.src.size())
+ self.src = fn(self.src, 0)
+ # print("self.src.size() after fn:", self.src.size())
+ tmp = []
+ # 我自己新加的,原来的presum这里对previous_layer_inputs没有做处理，导致后面size not match
+ if self.previous_input is not None:
+ for i in range(len(self.previous_layer_inputs)):
+ # print("map_batch_fn", self.previous_layer_inputs[i].size())
+
+ tmp.append(fn(self.previous_layer_inputs[i], 0))
+ self.previous_layer_inputs = tmp
+
+ if self.cache is not None:
+ _recursive_map(self.cache)
+=======
+"""
+Implementation of "Attention is All You Need"
+"""
+
+import torch
+import torch.nn as nn
+import numpy as np
+
+from models.encoder import PositionalEncoding
+from models.neural import MultiHeadedAttention, PositionwiseFeedForward, DecoderState
+
+MAX_SIZE = 5000
+
+
+class TransformerDecoderLayer(nn.Module):
+ """
+ Args:
+ d_model (int): the dimension of keys/values/queries in
+ MultiHeadedAttention, also the input size of
+ the first-layer of the PositionwiseFeedForward.
+ heads (int): the number of heads for MultiHeadedAttention.
+ d_ff (int): the second-layer of the PositionwiseFeedForward.
+ dropout (float): dropout probability(0-1.0).
+ self_attn_type (string): type of self-attention scaled-dot, average
+ """
+
+ def __init__(self, d_model, heads, d_ff, dropout):
+ super(TransformerDecoderLayer, self).__init__()
+
+ self.self_attn = MultiHeadedAttention(
+ heads, d_model, dropout=dropout)
+
+ self.context_attn = MultiHeadedAttention(
+ heads, d_model, dropout=dropout)
+ self.feed_forward = PositionwiseFeedForward(d_model, d_ff, dropout)
+ self.layer_norm_1 = nn.LayerNorm(d_model, eps=1e-6)
+ self.layer_norm_2 = nn.LayerNorm(d_model, eps=1e-6)
+ self.drop = nn.Dropout(dropout)
+ mask = self._get_attn_subsequent_mask(MAX_SIZE)
+ # Register self.mask as a buffer in TransformerDecoderLayer, so
+ # it gets TransformerDecoderLayer's cuda behavior automatically.
+ self.register_buffer('mask', mask)
+
+ def forward(self, inputs, memory_bank, src_pad_mask, tgt_pad_mask,
+ previous_input=None, layer_cache=None, step=None):
+ """
+ Args:
+ inputs (`FloatTensor`): `[batch_size x 1 x model_dim]`
+ memory_bank (`FloatTensor`): `[batch_size x src_len x model_dim]`
+ src_pad_mask (`LongTensor`): `[batch_size x 1 x src_len]`
+ tgt_pad_mask (`LongTensor`): `[batch_size x 1 x 1]`
+
+ Returns:
+ (`FloatTensor`, `FloatTensor`, `FloatTensor`):
+
+ * output `[batch_size x 1 x model_dim]`
+ * attn `[batch_size x 1 x src_len]`
+ * all_input `[batch_size x current_step x model_dim]`
+
+ """
+
+ dec_mask = torch.gt(tgt_pad_mask +
+ self.mask[:, :tgt_pad_mask.size(1),
+ :tgt_pad_mask.size(1)], 0)
+ # print("TransformerDecoderLayer.forward: ")
+ # print(inputs.size())
+ input_norm = self.layer_norm_1(inputs)
+ all_input = input_norm
+ if previous_input is not None:
+ # print(previous_input.size(), input_norm.size())
+ all_input = torch.cat((previous_input, input_norm), dim=1)
+ dec_mask = None
+
+ query = self.self_attn(all_input, all_input, input_norm,
+ mask=dec_mask,
+ layer_cache=layer_cache,
+ type="self")
+
+ query = self.drop(query) + inputs
+
+ query_norm = self.layer_norm_2(query)
+ mid = self.context_attn(memory_bank, memory_bank, query_norm,
+ mask=src_pad_mask,
+ layer_cache=layer_cache,
+ type="context")
+
+ output = self.feed_forward(self.drop(mid) + query)
+ return output, all_input
+ # return output
+
+ def _get_attn_subsequent_mask(self, size):
+ """
+ Get an attention mask to avoid using the subsequent info.
+
+ Args:
+ size: int
+
+ Returns:
+ (`LongTensor`):
+
+ * subsequent_mask `[1 x size x size]`
+ """
+ attn_shape = (1, size, size)
+ subsequent_mask = np.triu(np.ones(attn_shape), k=1).astype('uint8')
+ subsequent_mask = torch.from_numpy(subsequent_mask)
+ return subsequent_mask
+
+
+class TransformerDecoder(nn.Module):
+ """
+ The Transformer decoder from "Attention is All You Need".
+
+
+ .. mermaid::
+
+ graph BT
+ A[input]
+ B[multi-head self-attn]
+ BB[multi-head src-attn]
+ C[feed forward]
+ O[output]
+ A --> B
+ B --> BB
+ BB --> C
+ C --> O
+
+
+ Args:
+ num_layers (int): number of encoder layers.
+ d_model (int): size of the model
+ heads (int): number of heads
+ d_ff (int): size of the inner FF layer
+ dropout (float): dropout parameters
+ embeddings (:obj:`onmt.modules.Embeddings`):
+ embeddings to use, should have positional encodings
+ attn_type (str): if using a seperate copy attention
+ """
+
+ def __init__(self, num_layers, d_model, heads, d_ff, dropout, embeddings):
+ super(TransformerDecoder, self).__init__()
+
+ # Basic attributes.
+ self.decoder_type = 'transformer'
+ self.num_layers = num_layers
+ self.embeddings = embeddings
+ self.pos_emb = PositionalEncoding(dropout, self.embeddings.embedding_dim)
+
+ # Build TransformerDecoder.
+ self.transformer_layers = nn.ModuleList(
+ [TransformerDecoderLayer(d_model, heads, d_ff, dropout)
+ for _ in range(num_layers)])
+
+ self.layer_norm = nn.LayerNorm(d_model, eps=1e-6)
+
+ def forward(self, tgt, memory_bank, state, memory_lengths=None,
+ step=None, cache=None, memory_masks=None):
+ """
+ See :obj:`onmt.modules.RNNDecoderBase.forward()`
+ """
+
+ src_words = state.src
+ tgt_words = tgt
+ # print("tgt size: ", tgt.size())
+ src_batch, src_len = src_words.size()
+ tgt_batch, tgt_len = tgt_words.size()
+
+ # Run the forward pass of the TransformerDecoder.
+ # emb = self.embeddings(tgt, step=step)
+ emb = self.embeddings(tgt)
+ assert emb.dim() == 3 # len x batch x embedding_dim
+
+ output = self.pos_emb(emb, step)
+ # print("output before for roop: ", output.size())
+
+ src_memory_bank = memory_bank
+ padding_idx = self.embeddings.padding_idx
+ tgt_pad_mask = tgt_words.data.eq(padding_idx).unsqueeze(1) \
+ .expand(tgt_batch, tgt_len, tgt_len)
+
+ if (not memory_masks is None):
+ src_len = memory_masks.size(-1)
+ src_pad_mask = memory_masks.expand(src_batch, tgt_len, src_len)
+
+ else:
+ src_pad_mask = src_words.data.eq(padding_idx).unsqueeze(1) \
+ .expand(src_batch, tgt_len, src_len)
+
+ if state.cache is None:
+ saved_inputs = []
+ # print("state.cache:", state.cache)
+ for i in range(self.num_layers):
+ prev_layer_input = None
+ if state.cache is None:
+ if state.previous_input is not None:
+ # print("state.previous_input: ", state.previous_input.size())
+ # print("state.previous_layer_inputs[i]:", state.previous_layer_inputs[i].size())
+ prev_layer_input = state.previous_layer_inputs[i]
+ output, all_input \
+ = self.transformer_layers[i](
+ output, src_memory_bank,
+ src_pad_mask, tgt_pad_mask,
+ previous_input=prev_layer_input,
+ layer_cache=state.cache["layer_{}".format(i)]
+ if state.cache is not None else None,
+ step=step)
+ # if prev_layer_input is None:
+ # print("prev_layer_input is None")
+ # else:
+ # print("output size: {} previous_input_size: {}".format(output.size(), prev_layer_input.size()))
+ if state.cache is None:
+ saved_inputs.append(all_input)
+
+ if state.cache is None:
+ saved_inputs = torch.stack(saved_inputs)
+
+ output = self.layer_norm(output)
+
+ # Process the result and update the attentions.
+
+ if state.cache is None:
+ state = state.update_state(tgt, saved_inputs)
+
+ return output, state
+
+ def init_decoder_state(self, src, memory_bank,
+ with_cache=False):
+ """ Init decoder state """
+ state = TransformerDecoderState(src)
+ if with_cache:
+ state._init_cache(memory_bank, self.num_layers)
+ return state
+
+
+class TransformerDecoderState(DecoderState):
+ """ Transformer Decoder state base class """
+
+ def __init__(self, src):
+ """
+ Args:
+ src (FloatTensor): a sequence of source words tensors
+ with optional feature tensors, of size (len x batch).
+ """
+ self.src = src
+ self.previous_input = None
+ self.previous_layer_inputs = None
+ self.cache = None
+
+ @property
+ def _all(self):
+ """
+ Contains attributes that need to be updated in self.beam_update().
+ """
+ if (self.previous_input is not None
+ and self.previous_layer_inputs is not None):
+ return (self.previous_input,
+ self.previous_layer_inputs,
+ self.src)
+ else:
+ return (self.src,)
+
+ def detach(self):
+ if self.previous_input is not None:
+ self.previous_input = self.previous_input.detach()
+ if self.previous_layer_inputs is not None:
+ self.previous_layer_inputs = self.previous_layer_inputs.detach()
+ self.src = self.src.detach()
+
+ def update_state(self, new_input, previous_layer_inputs):
+ state = TransformerDecoderState(self.src)
+ state.previous_input = new_input
+ state.previous_layer_inputs = previous_layer_inputs
+ return state
+
+ def _init_cache(self, memory_bank, num_layers):
+ self.cache = {}
+
+ for l in range(num_layers):
+ layer_cache = {
+ "memory_keys": None,
+ "memory_values": None
+ }
+ layer_cache["self_keys"] = None
+ layer_cache["self_values"] = None
+ self.cache["layer_{}".format(l)] = layer_cache
+
+ def repeat_beam_size_times(self, beam_size):
+ """ Repeat beam_size times along batch dimension. """
+ self.src = self.src.data.repeat(1, beam_size, 1)
+
+ def map_batch_fn(self, fn):
+ def _recursive_map(struct, batch_dim=0):
+ for k, v in struct.items():
+ if v is not None:
+ if isinstance(v, dict):
+ _recursive_map(v)
+ else:
+ struct[k] = fn(v, batch_dim)
+
+ # print("self.src.size()", self.src.size())
+ self.src = fn(self.src, 0)
+ # print("self.src.size() after fn:", self.src.size())
+ tmp = []
+ # 我自己新加的,原来的presum这里对previous_layer_inputs没有做处理，导致后面size not match
+ if self.previous_input is not None:
+ for i in range(len(self.previous_layer_inputs)):
+ # print("map_batch_fn", self.previous_layer_inputs[i].size())
+
+ tmp.append(fn(self.previous_layer_inputs[i], 0))
+ self.previous_layer_inputs = tmp
+
+ if self.cache is not None:
+ _recursive_map(self.cache)
+>>>>>>> bcb618ae9faf5d35f3147b05b34268a73808238a
diff --git a/fastSum/PreSum/models/encoder.py b/fastSum/PreSum/models/encoder.py
new file mode 100644
index 0000000000000000000000000000000000000000..00be85799a5feafd89671abc170a47bddaf2867a
--- /dev/null
+++ b/fastSum/PreSum/models/encoder.py
@@ -0,0 +1,209 @@
+<<<<<<< HEAD
+import math
+
+import torch
+import torch.nn as nn
+
+from models.neural import MultiHeadedAttention, PositionwiseFeedForward
+
+
+class Classifier(nn.Module):
+ def __init__(self, hidden_size):
+ super(Classifier, self).__init__()
+ self.linear1 = nn.Linear(hidden_size, 1)
+ self.sigmoid = nn.Sigmoid()
+
+ def forward(self, x, mask_cls):
+ h = self.linear1(x).squeeze(-1)
+ sent_scores = self.sigmoid(h) * mask_cls.float()
+ return sent_scores
+
+
+class PositionalEncoding(nn.Module):
+
+ def __init__(self, dropout, dim, max_len=5000):
+ pe = torch.zeros(max_len, dim)
+ position = torch.arange(0, max_len).unsqueeze(1)
+ div_term = torch.exp((torch.arange(0, dim, 2, dtype=torch.float) *
+ -(math.log(10000.0) / dim)))
+ pe[:, 0::2] = torch.sin(position.float() * div_term)
+ pe[:, 1::2] = torch.cos(position.float() * div_term)
+ pe = pe.unsqueeze(0)
+ super(PositionalEncoding, self).__init__()
+ self.register_buffer('pe', pe)
+ self.dropout = nn.Dropout(p=dropout)
+ self.dim = dim
+
+ def forward(self, emb, step=None):
+ emb = emb * math.sqrt(self.dim)
+ if (step):
+ emb = emb + self.pe[:, step][:, None, :]
+
+ else:
+ emb = emb + self.pe[:, :emb.size(1)]
+ emb = self.dropout(emb)
+ return emb
+
+ def get_emb(self, emb):
+ return self.pe[:, :emb.size(1)]
+
+
+class TransformerEncoderLayer(nn.Module):
+ def __init__(self, d_model, heads, d_ff, dropout):
+ super(TransformerEncoderLayer, self).__init__()
+
+ self.self_attn = MultiHeadedAttention(
+ heads, d_model, dropout=dropout)
+ self.feed_forward = PositionwiseFeedForward(d_model, d_ff, dropout)
+ self.layer_norm = nn.LayerNorm(d_model, eps=1e-6)
+ self.dropout = nn.Dropout(dropout)
+
+ def forward(self, iter, query, inputs, mask):
+ if (iter != 0):
+ input_norm = self.layer_norm(inputs)
+ else:
+ input_norm = inputs
+
+ mask = mask.unsqueeze(1)
+ context = self.self_attn(input_norm, input_norm, input_norm,
+ mask=mask)
+ out = self.dropout(context) + inputs
+ return self.feed_forward(out)
+
+
+class ExtTransformerEncoder(nn.Module):
+ def __init__(self, d_model, d_ff, heads, dropout, num_inter_layers=0):
+ super(ExtTransformerEncoder, self).__init__()
+ self.d_model = d_model
+ self.num_inter_layers = num_inter_layers
+ self.pos_emb = PositionalEncoding(dropout, d_model)
+ self.transformer_inter = nn.ModuleList(
+ [TransformerEncoderLayer(d_model, heads, d_ff, dropout)
+ for _ in range(num_inter_layers)])
+ self.dropout = nn.Dropout(dropout)
+ self.layer_norm = nn.LayerNorm(d_model, eps=1e-6)
+ self.wo = nn.Linear(d_model, 1, bias=True)
+ self.sigmoid = nn.Sigmoid()
+
+ def forward(self, top_vecs, mask):
+ """ See :obj:`EncoderBase.forward()`"""
+
+ batch_size, n_sents = top_vecs.size(0), top_vecs.size(1)
+ pos_emb = self.pos_emb.pe[:, :n_sents]
+ x = top_vecs * mask[:, :, None].float()
+ x = x + pos_emb
+
+ for i in range(self.num_inter_layers):
+ x = self.transformer_inter[i](i, x, x, 1 - mask) # all_sents * max_tokens * dim
+
+ x = self.layer_norm(x)
+ sent_scores = self.sigmoid(self.wo(x))
+ sent_scores = sent_scores.squeeze(-1) * mask.float()
+
+ return sent_scores
+
+=======
+import math
+
+import torch
+import torch.nn as nn
+
+from models.neural import MultiHeadedAttention, PositionwiseFeedForward
+
+
+class Classifier(nn.Module):
+ def __init__(self, hidden_size):
+ super(Classifier, self).__init__()
+ self.linear1 = nn.Linear(hidden_size, 1)
+ self.sigmoid = nn.Sigmoid()
+
+ def forward(self, x, mask_cls):
+ h = self.linear1(x).squeeze(-1)
+ sent_scores = self.sigmoid(h) * mask_cls.float()
+ return sent_scores
+
+
+class PositionalEncoding(nn.Module):
+
+ def __init__(self, dropout, dim, max_len=5000):
+ pe = torch.zeros(max_len, dim)
+ position = torch.arange(0, max_len).unsqueeze(1)
+ div_term = torch.exp((torch.arange(0, dim, 2, dtype=torch.float) *
+ -(math.log(10000.0) / dim)))
+ pe[:, 0::2] = torch.sin(position.float() * div_term)
+ pe[:, 1::2] = torch.cos(position.float() * div_term)
+ pe = pe.unsqueeze(0)
+ super(PositionalEncoding, self).__init__()
+ self.register_buffer('pe', pe)
+ self.dropout = nn.Dropout(p=dropout)
+ self.dim = dim
+
+ def forward(self, emb, step=None):
+ emb = emb * math.sqrt(self.dim)
+ if (step):
+ emb = emb + self.pe[:, step][:, None, :]
+
+ else:
+ emb = emb + self.pe[:, :emb.size(1)]
+ emb = self.dropout(emb)
+ return emb
+
+ def get_emb(self, emb):
+ return self.pe[:, :emb.size(1)]
+
+
+class TransformerEncoderLayer(nn.Module):
+ def __init__(self, d_model, heads, d_ff, dropout):
+ super(TransformerEncoderLayer, self).__init__()
+
+ self.self_attn = MultiHeadedAttention(
+ heads, d_model, dropout=dropout)
+ self.feed_forward = PositionwiseFeedForward(d_model, d_ff, dropout)
+ self.layer_norm = nn.LayerNorm(d_model, eps=1e-6)
+ self.dropout = nn.Dropout(dropout)
+
+ def forward(self, iter, query, inputs, mask):
+ if (iter != 0):
+ input_norm = self.layer_norm(inputs)
+ else:
+ input_norm = inputs
+
+ mask = mask.unsqueeze(1)
+ context = self.self_attn(input_norm, input_norm, input_norm,
+ mask=mask)
+ out = self.dropout(context) + inputs
+ return self.feed_forward(out)
+
+
+class ExtTransformerEncoder(nn.Module):
+ def __init__(self, d_model, d_ff, heads, dropout, num_inter_layers=0):
+ super(ExtTransformerEncoder, self).__init__()
+ self.d_model = d_model
+ self.num_inter_layers = num_inter_layers
+ self.pos_emb = PositionalEncoding(dropout, d_model)
+ self.transformer_inter = nn.ModuleList(
+ [TransformerEncoderLayer(d_model, heads, d_ff, dropout)
+ for _ in range(num_inter_layers)])
+ self.dropout = nn.Dropout(dropout)
+ self.layer_norm = nn.LayerNorm(d_model, eps=1e-6)
+ self.wo = nn.Linear(d_model, 1, bias=True)
+ self.sigmoid = nn.Sigmoid()
+
+ def forward(self, top_vecs, mask):
+ """ See :obj:`EncoderBase.forward()`"""
+
+ batch_size, n_sents = top_vecs.size(0), top_vecs.size(1)
+ pos_emb = self.pos_emb.pe[:, :n_sents]
+ x = top_vecs * mask[:, :, None].float()
+ x = x + pos_emb
+
+ for i in range(self.num_inter_layers):
+ x = self.transformer_inter[i](i, x, x, 1 - mask) # all_sents * max_tokens * dim
+
+ x = self.layer_norm(x)
+ sent_scores = self.sigmoid(self.wo(x))
+ sent_scores = sent_scores.squeeze(-1) * mask.float()
+
+ return sent_scores
+
+>>>>>>> bcb618ae9faf5d35f3147b05b34268a73808238a
diff --git a/fastSum/PreSum/models/model_builder.py b/fastSum/PreSum/models/model_builder.py
new file mode 100644
index 0000000000000000000000000000000000000000..7380977a965bc75d079838f75fa6cbf74a06e3d7
--- /dev/null
+++ b/fastSum/PreSum/models/model_builder.py
@@ -0,0 +1,673 @@
+<<<<<<< HEAD
+import copy
+
+import torch
+import torch.nn as nn
+from pytorch_transformers import BertModel, BertConfig
+from torch.nn.init import xavier_uniform_
+
+from models.decoder import TransformerDecoder
+from models.encoder import Classifier, ExtTransformerEncoder
+from models.optimizers import Optimizer
+# from models.predictor import build_predictor
+from others.utils import rouge_results_to_str, test_rouge, tile
+from translate.beam import GNMTGlobalScorer
+
+
+def get_generator(vocab_size, dec_hidden_size):
+ gen_func = nn.LogSoftmax(dim=-1)
+ generator = nn.Sequential(
+ nn.Linear(dec_hidden_size, vocab_size),
+ gen_func
+ )
+ # generator.to(device)
+
+ return generator
+
+
+class Bert(nn.Module):
+ def __init__(self, large, temp_dir, finetune=False):
+ super(Bert, self).__init__()
+ if (large):
+ self.model = BertModel.from_pretrained('bert-large-uncased', cache_dir=temp_dir)
+ else:
+ self.model = BertModel.from_pretrained('bert-base-uncased', cache_dir=temp_dir)
+
+ self.finetune = finetune
+
+ def forward(self, x, segs, mask):
+ if (self.finetune):
+ top_vec, _ = self.model(x, segs, attention_mask=mask)
+ else:
+ self.eval()
+ with torch.no_grad():
+ top_vec, _ = self.model(x, segs, attention_mask=mask)
+ return top_vec
+
+
+class ExtSummarizer(nn.Module):
+ def __init__(self, args, checkpoint):
+ super(ExtSummarizer, self).__init__()
+ self.args = args
+ self.bert = Bert(args.large, args.temp_dir, args.finetune_bert)
+
+ self.ext_layer = ExtTransformerEncoder(self.bert.model.config.hidden_size, args.ext_ff_size, args.ext_heads,
+ args.ext_dropout, args.ext_layers)
+ if (args.encoder == 'baseline'):
+ bert_config = BertConfig(self.bert.model.config.vocab_size, hidden_size=args.ext_hidden_size,
+ num_hidden_layers=args.ext_layers, num_attention_heads=args.ext_heads,
+ intermediate_size=args.ext_ff_size)
+ self.bert.model = BertModel(bert_config)
+ self.ext_layer = Classifier(self.bert.model.config.hidden_size)
+
+ if (args.max_pos > 512):
+ my_pos_embeddings = nn.Embedding(args.max_pos, self.bert.model.config.hidden_size)
+ my_pos_embeddings.weight.data[:512] = self.bert.model.embeddings.position_embeddings.weight.data
+ my_pos_embeddings.weight.data[512:] = self.bert.model.embeddings.position_embeddings.weight.data[-1][None,
+ :].repeat(args.max_pos - 512, 1)
+ self.bert.model.embeddings.position_embeddings = my_pos_embeddings
+
+ if checkpoint is not None:
+ self.load_state_dict(checkpoint['model'], strict=True)
+ else:
+ if args.param_init != 0.0:
+ for p in self.ext_layer.parameters():
+ p.data.uniform_(-args.param_init, args.param_init)
+ if args.param_init_glorot:
+ for p in self.ext_layer.parameters():
+ if p.dim() > 1:
+ xavier_uniform_(p)
+
+ def forward(self, text_ids, segment_ids, cls_ids):
+ input_mask = 1 - (text_ids == 0).long()
+ mask_cls = 1 - (cls_ids == -1).to(torch.uint8)
+ assert input_mask.size() == text_ids.size()
+ assert mask_cls.size() == cls_ids.size()
+
+ top_vec = self.bert(text_ids, segment_ids, input_mask)
+ sents_vec = top_vec[torch.arange(top_vec.size(0)).unsqueeze(1), cls_ids]
+ sents_vec = sents_vec * mask_cls[:, :, None].float()
+ sent_scores = self.ext_layer(sents_vec, mask_cls).squeeze(-1)
+ return {'pred': sent_scores, 'mask': mask_cls}
+
+
+class AbsSummarizer(nn.Module):
+ def __init__(self, args, checkpoint=None, bert_from_extractive=None, symbols=None, tokenizer=None):
+ super(AbsSummarizer, self).__init__()
+ self.args = args
+ # self.device = device
+ self.bert = Bert(args.large, args.temp_dir, args.finetune_bert)
+ self.global_scorer = GNMTGlobalScorer(args.alpha, length_penalty='wu')
+ self.symbols = symbols
+ self.start_token = symbols['BOS']
+ self.end_token = symbols['EOS']
+ self.vocab = tokenizer
+
+ if bert_from_extractive is not None:
+ self.bert.model.load_state_dict(
+ dict([(n[11:], p) for n, p in bert_from_extractive.items() if n.startswith('bert.model')]), strict=True)
+
+ if (args.encoder == 'baseline'):
+ bert_config = BertConfig(self.bert.model.config.vocab_size, hidden_size=args.enc_hidden_size,
+ num_hidden_layers=args.enc_layers, num_attention_heads=8,
+ intermediate_size=args.enc_ff_size,
+ hidden_dropout_prob=args.enc_dropout,
+ attention_probs_dropout_prob=args.enc_dropout)
+ self.bert.model = BertModel(bert_config)
+
+ if (args.max_pos > 512):
+ my_pos_embeddings = nn.Embedding(args.max_pos, self.bert.model.config.hidden_size)
+ my_pos_embeddings.weight.data[:512] = self.bert.model.embeddings.position_embeddings.weight.data
+ my_pos_embeddings.weight.data[512:] = self.bert.model.embeddings.position_embeddings.weight.data[-1][None,
+ :].repeat(args.max_pos - 512, 1)
+ self.bert.model.embeddings.position_embeddings = my_pos_embeddings
+ self.vocab_size = self.bert.model.config.vocab_size
+ tgt_embeddings = nn.Embedding(self.vocab_size, self.bert.model.config.hidden_size, padding_idx=0)
+ if (self.args.share_emb):
+ tgt_embeddings.weight = copy.deepcopy(self.bert.model.embeddings.word_embeddings.weight)
+
+ self.decoder = TransformerDecoder(
+ self.args.dec_layers,
+ self.args.dec_hidden_size, heads=self.args.dec_heads,
+ d_ff=self.args.dec_ff_size, dropout=self.args.dec_dropout, embeddings=tgt_embeddings)
+
+ self.generator = get_generator(self.vocab_size, self.args.dec_hidden_size)
+ self.generator[0].weight = self.decoder.embeddings.weight
+
+ if checkpoint is not None:
+ self.load_state_dict(checkpoint['model'], strict=True)
+ else:
+ for module in self.decoder.modules():
+ if isinstance(module, (nn.Linear, nn.Embedding)):
+ module.weight.data.normal_(mean=0.0, std=0.02)
+ elif isinstance(module, nn.LayerNorm):
+ module.bias.data.zero_()
+ module.weight.data.fill_(1.0)
+ if isinstance(module, nn.Linear) and module.bias is not None:
+ module.bias.data.zero_()
+ for p in self.generator.parameters():
+ if p.dim() > 1:
+ xavier_uniform_(p)
+ else:
+ p.data.zero_()
+ if (args.use_bert_emb):
+ tgt_embeddings = nn.Embedding(self.vocab_size, self.bert.model.config.hidden_size, padding_idx=0)
+ tgt_embeddings.weight = copy.deepcopy(self.bert.model.embeddings.word_embeddings.weight)
+ self.decoder.embeddings = tgt_embeddings
+ self.generator[0].weight = self.decoder.embeddings.weight
+
+ # self.to(device)
+
+ def forward(self, text_ids, summary_ids, segment_ids):
+ # 默认设定pad id为0
+ mask_src = 1 - (text_ids == 0).long()
+ top_vec = self.bert(text_ids, segment_ids, mask_src)
+ dec_state = self.decoder.init_decoder_state(text_ids, top_vec)
+ # print("AbsSummarizer.forward: ")
+ # print(summary_ids[:, :-1].size(), top_vec.size())
+ pred, state = self.decoder(summary_ids[:, :-1], top_vec, dec_state)
+ return {"pred": pred}
+
+ def predict(self, text_ids, segment_ids, summary_ids=None):
+ # 当predict在训练的过程中的validation的过程被调用，则返回forward的结果
+ if self.args.mode == "train":
+ return self.forward(text_ids, summary_ids, segment_ids)
+
+ min_length = self.args.min_length
+ max_length = self.args.max_length
+ beam_size = self.args.beam_size
+ # batch_size = self.args.batch_size
+ batch_size = text_ids.size()[0]
+
+ # 默认设定pad id为0
+ mask_src = 1 - (text_ids == 0).long()
+ top_vec = self.bert(text_ids, segment_ids, mask_src)
+ dec_states = self.decoder.init_decoder_state(text_ids, top_vec)
+ device = top_vec.device
+
+ # Tile states and memory beam_size times.
+ dec_states.map_batch_fn(
+ lambda state, dim: tile(state, beam_size, dim=dim))
+ src_features = tile(top_vec, beam_size, dim=0)
+ batch_offset = torch.arange(
+ batch_size, dtype=torch.long, device=device)
+ beam_offset = torch.arange(
+ 0,
+ batch_size * beam_size,
+ step=beam_size,
+ dtype=torch.long,
+ device=device)
+ alive_seq = torch.full(
+ [batch_size * beam_size, 1],
+ self.start_token,
+ dtype=torch.long,
+ device=device)
+ # print("AbsSummarizer.predict: ")
+ # print(alive_seq.size(), batch_size, beam_size)
+ # Give full probability to the first beam on the first step.
+ topk_log_probs = (
+ torch.tensor([0.0] + [float("-inf")] * (beam_size - 1),
+ device=device).repeat(batch_size))
+
+ # Structure that holds finished hypotheses.
+ hypotheses = [[] for _ in range(batch_size)] # noqa: F812
+
+ results = {}
+ results["predictions"] = [[] for _ in range(batch_size)] # noqa: F812
+ results["scores"] = [[] for _ in range(batch_size)] # noqa: F812
+ # results["gold_score"] = [0] * batch_size
+
+ for step in range(max_length):
+ # print("AbsSummarizer.predict:")
+ # print(alive_seq.size())
+ # print("alive seq: ", alive_seq.size())
+ decoder_input = alive_seq[:, -1].view(1, -1)
+
+ # Decoder forward.
+ decoder_input = decoder_input.transpose(0, 1)
+ # print("decoder input: ", decoder_input.size())
+
+ # print(decoder_input.size(), src_features.size())
+ dec_out, dec_states = self.decoder(decoder_input, src_features, dec_states,
+ step=step)
+
+ # Generator forward.
+ log_probs = self.generator.forward(dec_out.transpose(0, 1).squeeze(0))
+ vocab_size = log_probs.size(-1)
+
+ if step < min_length:
+ log_probs[:, self.end_token] = -1e20
+
+ # Multiply probs by the beam probability.
+ log_probs += topk_log_probs.view(-1).unsqueeze(1)
+
+ alpha = self.global_scorer.alpha
+ length_penalty = ((5.0 + (step + 1)) / 6.0) ** alpha
+
+ # Flatten probs into a list of possibilities.
+ curr_scores = log_probs / length_penalty
+
+ if (self.args.block_trigram):
+ cur_len = alive_seq.size(1)
+ if (cur_len > 3):
+ for i in range(alive_seq.size(0)):
+ fail = False
+ words = [int(w) for w in alive_seq[i]]
+ words = [self.vocab.ids_to_tokens[w] for w in words]
+ words = ' '.join(words).replace(' ##', '').split()
+ if (len(words) <= 3):
+ continue
+ trigrams = [(words[i - 1], words[i], words[i + 1]) for i in range(1, len(words) - 1)]
+ trigram = tuple(trigrams[-1])
+ if trigram in trigrams[:-1]:
+ fail = True
+ if fail:
+ curr_scores[i] = -10e20
+
+ curr_scores = curr_scores.reshape(-1, beam_size * vocab_size)
+ topk_scores, topk_ids = curr_scores.topk(beam_size, dim=-1)
+
+ # Recover log probs.
+ topk_log_probs = topk_scores * length_penalty
+
+ # Resolve beam origin and true word ids.
+ topk_beam_index = topk_ids.div(vocab_size)
+ topk_ids = topk_ids.fmod(vocab_size)
+
+ # Map beam_index to batch_index in the flat representation.
+ batch_index = (
+ topk_beam_index
+ + beam_offset[:topk_beam_index.size(0)].unsqueeze(1))
+ select_indices = batch_index.view(-1)
+
+ # Append last prediction.
+ alive_seq = torch.cat(
+ [alive_seq.index_select(0, select_indices),
+ topk_ids.view(-1, 1)], -1)
+ # print("alive_seq after cat: ", alive_seq.size())
+
+ is_finished = topk_ids.eq(self.end_token)
+ if step + 1 == max_length:
+ is_finished.fill_(1)
+ # End condition is top beam is finished.
+ end_condition = is_finished[:, 0].eq(1)
+ # Save finished hypotheses.
+ if is_finished.any():
+ predictions = alive_seq.view(-1, beam_size, alive_seq.size(-1))
+ for i in range(is_finished.size(0)):
+ b = batch_offset[i]
+ if end_condition[i]:
+ is_finished[i].fill_(1)
+ finished_hyp = is_finished[i].nonzero().view(-1)
+ # Store finished hypotheses for this batch.
+ for j in finished_hyp:
+ hypotheses[b].append((
+ topk_scores[i, j],
+ predictions[i, j, 1:]))
+ # If the batch reached the end, save the n_best hypotheses.
+ if end_condition[i]:
+ best_hyp = sorted(
+ hypotheses[b], key=lambda x: x[0], reverse=True)
+ score, pred = best_hyp[0]
+
+ results["scores"][b].append(score)
+ results["predictions"][b].append(pred)
+ non_finished = end_condition.eq(0).nonzero().view(-1)
+ # print(non_finished)
+ # If all sentences are translated, no need to go further.
+ if len(non_finished) == 0:
+ break
+ # Remove finished batches for the next step.
+ topk_log_probs = topk_log_probs.index_select(0, non_finished)
+ batch_index = batch_index.index_select(0, non_finished)
+ batch_offset = batch_offset.index_select(0, non_finished)
+ alive_seq = predictions.index_select(0, non_finished) \
+ .view(-1, alive_seq.size(-1))
+ # print("alive_seq predictions.index_select ", alive_seq.size())
+ # Reorder states.
+ select_indices = batch_index.view(-1)
+ # print("select indices: {}".format(select_indices.size()))
+ # print("src_features before index_select:", src_features.size())
+ src_features = src_features.index_select(0, select_indices)
+ # print("src_features: {}".format(src_features.size()))
+ dec_states.map_batch_fn(
+ lambda state, dim: state.index_select(dim, select_indices))
+
+ return results
+=======
+import copy
+
+import torch
+import torch.nn as nn
+from pytorch_transformers import BertModel, BertConfig
+from torch.nn.init import xavier_uniform_
+
+from models.decoder import TransformerDecoder
+from models.encoder import Classifier, ExtTransformerEncoder
+from models.optimizers import Optimizer
+# from models.predictor import build_predictor
+from others.utils import rouge_results_to_str, test_rouge, tile
+from translate.beam import GNMTGlobalScorer
+
+
+def get_generator(vocab_size, dec_hidden_size):
+ gen_func = nn.LogSoftmax(dim=-1)
+ generator = nn.Sequential(
+ nn.Linear(dec_hidden_size, vocab_size),
+ gen_func
+ )
+ # generator.to(device)
+
+ return generator
+
+
+class Bert(nn.Module):
+ def __init__(self, large, temp_dir, finetune=False):
+ super(Bert, self).__init__()
+ if (large):
+ self.model = BertModel.from_pretrained('bert-large-uncased', cache_dir=temp_dir)
+ else:
+ self.model = BertModel.from_pretrained('bert-base-uncased', cache_dir=temp_dir)
+
+ self.finetune = finetune
+
+ def forward(self, x, segs, mask):
+ if (self.finetune):
+ top_vec, _ = self.model(x, segs, attention_mask=mask)
+ else:
+ self.eval()
+ with torch.no_grad():
+ top_vec, _ = self.model(x, segs, attention_mask=mask)
+ return top_vec
+
+
+class ExtSummarizer(nn.Module):
+ def __init__(self, args, checkpoint):
+ super(ExtSummarizer, self).__init__()
+ self.args = args
+ self.bert = Bert(args.large, args.temp_dir, args.finetune_bert)
+
+ self.ext_layer = ExtTransformerEncoder(self.bert.model.config.hidden_size, args.ext_ff_size, args.ext_heads,
+ args.ext_dropout, args.ext_layers)
+ if (args.encoder == 'baseline'):
+ bert_config = BertConfig(self.bert.model.config.vocab_size, hidden_size=args.ext_hidden_size,
+ num_hidden_layers=args.ext_layers, num_attention_heads=args.ext_heads,
+ intermediate_size=args.ext_ff_size)
+ self.bert.model = BertModel(bert_config)
+ self.ext_layer = Classifier(self.bert.model.config.hidden_size)
+
+ if (args.max_pos > 512):
+ my_pos_embeddings = nn.Embedding(args.max_pos, self.bert.model.config.hidden_size)
+ my_pos_embeddings.weight.data[:512] = self.bert.model.embeddings.position_embeddings.weight.data
+ my_pos_embeddings.weight.data[512:] = self.bert.model.embeddings.position_embeddings.weight.data[-1][None,
+ :].repeat(args.max_pos - 512, 1)
+ self.bert.model.embeddings.position_embeddings = my_pos_embeddings
+
+ if checkpoint is not None:
+ self.load_state_dict(checkpoint['model'], strict=True)
+ else:
+ if args.param_init != 0.0:
+ for p in self.ext_layer.parameters():
+ p.data.uniform_(-args.param_init, args.param_init)
+ if args.param_init_glorot:
+ for p in self.ext_layer.parameters():
+ if p.dim() > 1:
+ xavier_uniform_(p)
+
+ def forward(self, text_ids, segment_ids, cls_ids):
+ input_mask = 1 - (text_ids == 0).long()
+ mask_cls = 1 - (cls_ids == -1).to(torch.uint8)
+ assert input_mask.size() == text_ids.size()
+ assert mask_cls.size() == cls_ids.size()
+
+ top_vec = self.bert(text_ids, segment_ids, input_mask)
+ sents_vec = top_vec[torch.arange(top_vec.size(0)).unsqueeze(1), cls_ids]
+ sents_vec = sents_vec * mask_cls[:, :, None].float()
+ sent_scores = self.ext_layer(sents_vec, mask_cls).squeeze(-1)
+ return {'pred': sent_scores, 'mask': mask_cls}
+
+
+class AbsSummarizer(nn.Module):
+ def __init__(self, args, checkpoint=None, bert_from_extractive=None, symbols=None, tokenizer=None):
+ super(AbsSummarizer, self).__init__()
+ self.args = args
+ # self.device = device
+ self.bert = Bert(args.large, args.temp_dir, args.finetune_bert)
+ self.global_scorer = GNMTGlobalScorer(args.alpha, length_penalty='wu')
+ self.symbols = symbols
+ self.start_token = symbols['BOS']
+ self.end_token = symbols['EOS']
+ self.vocab = tokenizer
+
+ if bert_from_extractive is not None:
+ self.bert.model.load_state_dict(
+ dict([(n[11:], p) for n, p in bert_from_extractive.items() if n.startswith('bert.model')]), strict=True)
+
+ if (args.encoder == 'baseline'):
+ bert_config = BertConfig(self.bert.model.config.vocab_size, hidden_size=args.enc_hidden_size,
+ num_hidden_layers=args.enc_layers, num_attention_heads=8,
+ intermediate_size=args.enc_ff_size,
+ hidden_dropout_prob=args.enc_dropout,
+ attention_probs_dropout_prob=args.enc_dropout)
+ self.bert.model = BertModel(bert_config)
+
+ if (args.max_pos > 512):
+ my_pos_embeddings = nn.Embedding(args.max_pos, self.bert.model.config.hidden_size)
+ my_pos_embeddings.weight.data[:512] = self.bert.model.embeddings.position_embeddings.weight.data
+ my_pos_embeddings.weight.data[512:] = self.bert.model.embeddings.position_embeddings.weight.data[-1][None,
+ :].repeat(args.max_pos - 512, 1)
+ self.bert.model.embeddings.position_embeddings = my_pos_embeddings
+ self.vocab_size = self.bert.model.config.vocab_size
+ tgt_embeddings = nn.Embedding(self.vocab_size, self.bert.model.config.hidden_size, padding_idx=0)
+ if (self.args.share_emb):
+ tgt_embeddings.weight = copy.deepcopy(self.bert.model.embeddings.word_embeddings.weight)
+
+ self.decoder = TransformerDecoder(
+ self.args.dec_layers,
+ self.args.dec_hidden_size, heads=self.args.dec_heads,
+ d_ff=self.args.dec_ff_size, dropout=self.args.dec_dropout, embeddings=tgt_embeddings)
+
+ self.generator = get_generator(self.vocab_size, self.args.dec_hidden_size)
+ self.generator[0].weight = self.decoder.embeddings.weight
+
+ if checkpoint is not None:
+ self.load_state_dict(checkpoint['model'], strict=True)
+ else:
+ for module in self.decoder.modules():
+ if isinstance(module, (nn.Linear, nn.Embedding)):
+ module.weight.data.normal_(mean=0.0, std=0.02)
+ elif isinstance(module, nn.LayerNorm):
+ module.bias.data.zero_()
+ module.weight.data.fill_(1.0)
+ if isinstance(module, nn.Linear) and module.bias is not None:
+ module.bias.data.zero_()
+ for p in self.generator.parameters():
+ if p.dim() > 1:
+ xavier_uniform_(p)
+ else:
+ p.data.zero_()
+ if (args.use_bert_emb):
+ tgt_embeddings = nn.Embedding(self.vocab_size, self.bert.model.config.hidden_size, padding_idx=0)
+ tgt_embeddings.weight = copy.deepcopy(self.bert.model.embeddings.word_embeddings.weight)
+ self.decoder.embeddings = tgt_embeddings
+ self.generator[0].weight = self.decoder.embeddings.weight
+
+ # self.to(device)
+
+ def forward(self, text_ids, summary_ids, segment_ids):
+ # 默认设定pad id为0
+ mask_src = 1 - (text_ids == 0).long()
+ top_vec = self.bert(text_ids, segment_ids, mask_src)
+ dec_state = self.decoder.init_decoder_state(text_ids, top_vec)
+ # print("AbsSummarizer.forward: ")
+ # print(summary_ids[:, :-1].size(), top_vec.size())
+ pred, state = self.decoder(summary_ids[:, :-1], top_vec, dec_state)
+ return {"pred": pred}
+
+ def predict(self, text_ids, segment_ids, summary_ids=None):
+ # 当predict在训练的过程中的validation的过程被调用，则返回forward的结果
+ if self.args.mode == "train":
+ return self.forward(text_ids, summary_ids, segment_ids)
+
+ min_length = self.args.min_length
+ max_length = self.args.max_length
+ beam_size = self.args.beam_size
+ # batch_size = self.args.batch_size
+ batch_size = text_ids.size()[0]
+
+ # 默认设定pad id为0
+ mask_src = 1 - (text_ids == 0).long()
+ top_vec = self.bert(text_ids, segment_ids, mask_src)
+ dec_states = self.decoder.init_decoder_state(text_ids, top_vec)
+ device = top_vec.device
+
+ # Tile states and memory beam_size times.
+ dec_states.map_batch_fn(
+ lambda state, dim: tile(state, beam_size, dim=dim))
+ src_features = tile(top_vec, beam_size, dim=0)
+ batch_offset = torch.arange(
+ batch_size, dtype=torch.long, device=device)
+ beam_offset = torch.arange(
+ 0,
+ batch_size * beam_size,
+ step=beam_size,
+ dtype=torch.long,
+ device=device)
+ alive_seq = torch.full(
+ [batch_size * beam_size, 1],
+ self.start_token,
+ dtype=torch.long,
+ device=device)
+ # print("AbsSummarizer.predict: ")
+ # print(alive_seq.size(), batch_size, beam_size)
+ # Give full probability to the first beam on the first step.
+ topk_log_probs = (
+ torch.tensor([0.0] + [float("-inf")] * (beam_size - 1),
+ device=device).repeat(batch_size))
+
+ # Structure that holds finished hypotheses.
+ hypotheses = [[] for _ in range(batch_size)] # noqa: F812
+
+ results = {}
+ results["predictions"] = [[] for _ in range(batch_size)] # noqa: F812
+ results["scores"] = [[] for _ in range(batch_size)] # noqa: F812
+ # results["gold_score"] = [0] * batch_size
+
+ for step in range(max_length):
+ # print("AbsSummarizer.predict:")
+ # print(alive_seq.size())
+ # print("alive seq: ", alive_seq.size())
+ decoder_input = alive_seq[:, -1].view(1, -1)
+
+ # Decoder forward.
+ decoder_input = decoder_input.transpose(0, 1)
+ # print("decoder input: ", decoder_input.size())
+
+ # print(decoder_input.size(), src_features.size())
+ dec_out, dec_states = self.decoder(decoder_input, src_features, dec_states,
+ step=step)
+
+ # Generator forward.
+ log_probs = self.generator.forward(dec_out.transpose(0, 1).squeeze(0))
+ vocab_size = log_probs.size(-1)
+
+ if step < min_length:
+ log_probs[:, self.end_token] = -1e20
+
+ # Multiply probs by the beam probability.
+ log_probs += topk_log_probs.view(-1).unsqueeze(1)
+
+ alpha = self.global_scorer.alpha
+ length_penalty = ((5.0 + (step + 1)) / 6.0) ** alpha
+
+ # Flatten probs into a list of possibilities.
+ curr_scores = log_probs / length_penalty
+
+ if (self.args.block_trigram):
+ cur_len = alive_seq.size(1)
+ if (cur_len > 3):
+ for i in range(alive_seq.size(0)):
+ fail = False
+ words = [int(w) for w in alive_seq[i]]
+ words = [self.vocab.ids_to_tokens[w] for w in words]
+ words = ' '.join(words).replace(' ##', '').split()
+ if (len(words) <= 3):
+ continue
+ trigrams = [(words[i - 1], words[i], words[i + 1]) for i in range(1, len(words) - 1)]
+ trigram = tuple(trigrams[-1])
+ if trigram in trigrams[:-1]:
+ fail = True
+ if fail:
+ curr_scores[i] = -10e20
+
+ curr_scores = curr_scores.reshape(-1, beam_size * vocab_size)
+ topk_scores, topk_ids = curr_scores.topk(beam_size, dim=-1)
+
+ # Recover log probs.
+ topk_log_probs = topk_scores * length_penalty
+
+ # Resolve beam origin and true word ids.
+ topk_beam_index = topk_ids.div(vocab_size)
+ topk_ids = topk_ids.fmod(vocab_size)
+
+ # Map beam_index to batch_index in the flat representation.
+ batch_index = (
+ topk_beam_index
+ + beam_offset[:topk_beam_index.size(0)].unsqueeze(1))
+ select_indices = batch_index.view(-1)
+
+ # Append last prediction.
+ alive_seq = torch.cat(
+ [alive_seq.index_select(0, select_indices),
+ topk_ids.view(-1, 1)], -1)
+ # print("alive_seq after cat: ", alive_seq.size())
+
+ is_finished = topk_ids.eq(self.end_token)
+ if step + 1 == max_length:
+ is_finished.fill_(1)
+ # End condition is top beam is finished.
+ end_condition = is_finished[:, 0].eq(1)
+ # Save finished hypotheses.
+ if is_finished.any():
+ predictions = alive_seq.view(-1, beam_size, alive_seq.size(-1))
+ for i in range(is_finished.size(0)):
+ b = batch_offset[i]
+ if end_condition[i]:
+ is_finished[i].fill_(1)
+ finished_hyp = is_finished[i].nonzero().view(-1)
+ # Store finished hypotheses for this batch.
+ for j in finished_hyp:
+ hypotheses[b].append((
+ topk_scores[i, j],
+ predictions[i, j, 1:]))
+ # If the batch reached the end, save the n_best hypotheses.
+ if end_condition[i]:
+ best_hyp = sorted(
+ hypotheses[b], key=lambda x: x[0], reverse=True)
+ score, pred = best_hyp[0]
+
+ results["scores"][b].append(score)
+ results["predictions"][b].append(pred)
+ non_finished = end_condition.eq(0).nonzero().view(-1)
+ # print(non_finished)
+ # If all sentences are translated, no need to go further.
+ if len(non_finished) == 0:
+ break
+ # Remove finished batches for the next step.
+ topk_log_probs = topk_log_probs.index_select(0, non_finished)
+ batch_index = batch_index.index_select(0, non_finished)
+ batch_offset = batch_offset.index_select(0, non_finished)
+ alive_seq = predictions.index_select(0, non_finished) \
+ .view(-1, alive_seq.size(-1))
+ # print("alive_seq predictions.index_select ", alive_seq.size())
+ # Reorder states.
+ select_indices = batch_index.view(-1)
+ # print("select indices: {}".format(select_indices.size()))
+ # print("src_features before index_select:", src_features.size())
+ src_features = src_features.index_select(0, select_indices)
+ # print("src_features: {}".format(src_features.size()))
+ dec_states.map_batch_fn(
+ lambda state, dim: state.index_select(dim, select_indices))
+
+ return results
+>>>>>>> bcb618ae9faf5d35f3147b05b34268a73808238a
diff --git a/fastSum/PreSum/models/neural.py b/fastSum/PreSum/models/neural.py
new file mode 100644
index 0000000000000000000000000000000000000000..301fd574fa691c81a64a8c9daf897b24c77da691
--- /dev/null
+++ b/fastSum/PreSum/models/neural.py
@@ -0,0 +1,957 @@
+<<<<<<< HEAD
+import math
+
+import torch
+import torch.nn as nn
+import torchsnooper
+
+def aeq(*args):
+ """
+ Assert all arguments have the same value
+ """
+ arguments = (arg for arg in args)
+ first = next(arguments)
+ assert all(arg == first for arg in arguments), \
+ "Not all arguments have the same value: " + str(args)
+
+
+def sequence_mask(lengths, max_len=None):
+ """
+ Creates a boolean mask from sequence lengths.
+ """
+ batch_size = lengths.numel()
+ max_len = max_len or lengths.max()
+ return (torch.arange(0, max_len)
+ .type_as(lengths)
+ .repeat(batch_size, 1)
+ .lt(lengths.unsqueeze(1)))
+
+
+def gelu(x):
+ return 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))
+
+
+""" Global attention modules (Luong / Bahdanau) """
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+class GlobalAttention(nn.Module):
+ """
+ Global attention takes a matrix and a query vector. It
+ then computes a parameterized convex combination of the matrix
+ based on the input query.
+
+ Constructs a unit mapping a query `q` of size `dim`
+ and a source matrix `H` of size `n x dim`, to an output
+ of size `dim`.
+
+
+ .. mermaid::
+
+ graph BT
+ A[Query]
+ subgraph RNN
+ C[H 1]
+ D[H 2]
+ E[H N]
+ end
+ F[Attn]
+ G[Output]
+ A --> F
+ C --> F
+ D --> F
+ E --> F
+ C -.-> G
+ D -.-> G
+ E -.-> G
+ F --> G
+
+ All models compute the output as
+ :math:`c = sum_{j=1}^{SeqLength} a_j H_j` where
+ :math:`a_j` is the softmax of a score function.
+ Then then apply a projection layer to [q, c].
+
+ However they
+ differ on how they compute the attention score.
+
+ * Luong Attention (dot, general):
+ * dot: :math:`score(H_j,q) = H_j^T q`
+ * general: :math:`score(H_j, q) = H_j^T W_a q`
+
+
+ * Bahdanau Attention (mlp):
+ * :math:`score(H_j, q) = v_a^T tanh(W_a q + U_a h_j)`
+
+
+ Args:
+ dim (int): dimensionality of query and key
+ coverage (bool): use coverage term
+ attn_type (str): type of attention to use, options [dot,general,mlp]
+
+ """
+
+ def __init__(self, dim, attn_type="dot"):
+ super(GlobalAttention, self).__init__()
+
+ self.dim = dim
+ assert attn_type in ["dot", "general", "mlp"], (
+ "Please select a valid attention type.")
+ self.attn_type = attn_type
+
+ if self.attn_type == "general":
+ self.linear_in = nn.Linear(dim, dim, bias=False)
+ elif self.attn_type == "mlp":
+ self.linear_context = nn.Linear(dim, dim, bias=False)
+ self.linear_query = nn.Linear(dim, dim, bias=True)
+ self.v = nn.Linear(dim, 1, bias=False)
+ # mlp wants it with bias
+ out_bias = self.attn_type == "mlp"
+ self.linear_out = nn.Linear(dim * 2, dim, bias=out_bias)
+
+
+ def score(self, h_t, h_s):
+ """
+ Args:
+ h_t (`FloatTensor`): sequence of queries `[batch x tgt_len x dim]`
+ h_s (`FloatTensor`): sequence of sources `[batch x src_len x dim]`
+
+ Returns:
+ :obj:`FloatTensor`:
+ raw attention scores (unnormalized) for each src index
+ `[batch x tgt_len x src_len]`
+
+ """
+
+ # Check input sizes
+ src_batch, src_len, src_dim = h_s.size()
+ tgt_batch, tgt_len, tgt_dim = h_t.size()
+
+ if self.attn_type in ["general", "dot"]:
+ if self.attn_type == "general":
+ h_t_ = h_t.view(tgt_batch * tgt_len, tgt_dim)
+ h_t_ = self.linear_in(h_t_)
+ h_t = h_t_.view(tgt_batch, tgt_len, tgt_dim)
+ h_s_ = h_s.transpose(1, 2)
+ # (batch, t_len, d) x (batch, d, s_len) --> (batch, t_len, s_len)
+ return torch.bmm(h_t, h_s_)
+ else:
+ dim = self.dim
+ wq = self.linear_query(h_t.view(-1, dim))
+ wq = wq.view(tgt_batch, tgt_len, 1, dim)
+ wq = wq.expand(tgt_batch, tgt_len, src_len, dim)
+
+ uh = self.linear_context(h_s.contiguous().view(-1, dim))
+ uh = uh.view(src_batch, 1, src_len, dim)
+ uh = uh.expand(src_batch, tgt_len, src_len, dim)
+
+ # (batch, t_len, s_len, d)
+ wquh = torch.tanh(wq + uh)
+
+ return self.v(wquh.view(-1, dim)).view(tgt_batch, tgt_len, src_len)
+
+ def forward(self, source, memory_bank, memory_lengths=None, memory_masks=None):
+ """
+
+ Args:
+ source (`FloatTensor`): query vectors `[batch x tgt_len x dim]`
+ memory_bank (`FloatTensor`): source vectors `[batch x src_len x dim]`
+ memory_lengths (`LongTensor`): the source context lengths `[batch]`
+ coverage (`FloatTensor`): None (not supported yet)
+
+ Returns:
+ (`FloatTensor`, `FloatTensor`):
+
+ * Computed vector `[tgt_len x batch x dim]`
+ * Attention distribtutions for each query
+ `[tgt_len x batch x src_len]`
+ """
+
+ # one step input
+ if source.dim() == 2:
+ one_step = True
+ source = source.unsqueeze(1)
+ else:
+ one_step = False
+
+ batch, source_l, dim = memory_bank.size()
+ batch_, target_l, dim_ = source.size()
+
+ # compute attention scores, as in Luong et al.
+ align = self.score(source, memory_bank)
+
+ if memory_masks is not None:
+ memory_masks = memory_masks.transpose(0,1)
+ memory_masks = memory_masks.transpose(1,2)
+ align.masked_fill_(1 - memory_masks.byte(), -float('inf'))
+
+ if memory_lengths is not None:
+ mask = sequence_mask(memory_lengths, max_len=align.size(-1))
+ mask = mask.unsqueeze(1) # Make it broadcastable.
+ align.masked_fill_(1 - mask, -float('inf'))
+
+ align_vectors = F.softmax(align.view(batch*target_l, source_l), -1)
+ align_vectors = align_vectors.view(batch, target_l, source_l)
+
+ c = torch.bmm(align_vectors, memory_bank)
+
+ # concatenate
+ concat_c = torch.cat([c, source], 2).view(batch*target_l, dim*2)
+ attn_h = self.linear_out(concat_c).view(batch, target_l, dim)
+ if self.attn_type in ["general", "dot"]:
+ attn_h = torch.tanh(attn_h)
+
+ if one_step:
+ attn_h = attn_h.squeeze(1)
+ align_vectors = align_vectors.squeeze(1)
+
+
+ else:
+ attn_h = attn_h.transpose(0, 1).contiguous()
+ align_vectors = align_vectors.transpose(0, 1).contiguous()
+
+ return attn_h, align_vectors
+
+
+class PositionwiseFeedForward(nn.Module):
+ """ A two-layer Feed-Forward-Network with residual layer norm.
+
+ Args:
+ d_model (int): the size of input for the first-layer of the FFN.
+ d_ff (int): the hidden layer size of the second-layer
+ of the FNN.
+ dropout (float): dropout probability in :math:`[0, 1)`.
+ """
+
+ def __init__(self, d_model, d_ff, dropout=0.1):
+ super(PositionwiseFeedForward, self).__init__()
+ self.w_1 = nn.Linear(d_model, d_ff)
+ self.w_2 = nn.Linear(d_ff, d_model)
+ self.layer_norm = nn.LayerNorm(d_model, eps=1e-6)
+ self.actv = gelu
+ self.dropout_1 = nn.Dropout(dropout)
+ self.dropout_2 = nn.Dropout(dropout)
+
+ def forward(self, x):
+ inter = self.dropout_1(self.actv(self.w_1(self.layer_norm(x))))
+ output = self.dropout_2(self.w_2(inter))
+ return output + x
+
+
+class MultiHeadedAttention(nn.Module):
+ """
+ Multi-Head Attention module from
+ "Attention is All You Need"
+ :cite:`DBLP:journals/corr/VaswaniSPUJGKP17`.
+
+ Similar to standard `dot` attention but uses
+ multiple attention distributions simulataneously
+ to select relevant items.
+
+ .. mermaid::
+
+ graph BT
+ A[key]
+ B[value]
+ C[query]
+ O[output]
+ subgraph Attn
+ D[Attn 1]
+ E[Attn 2]
+ F[Attn N]
+ end
+ A --> D
+ C --> D
+ A --> E
+ C --> E
+ A --> F
+ C --> F
+ D --> O
+ E --> O
+ F --> O
+ B --> O
+
+ Also includes several additional tricks.
+
+ Args:
+ head_count (int): number of parallel heads
+ model_dim (int): the dimension of keys/values/queries,
+ must be divisible by head_count
+ dropout (float): dropout parameter
+ """
+
+ def __init__(self, head_count, model_dim, dropout=0.1, use_final_linear=True):
+ assert model_dim % head_count == 0
+ self.dim_per_head = model_dim // head_count
+ self.model_dim = model_dim
+
+ super(MultiHeadedAttention, self).__init__()
+ self.head_count = head_count
+
+ self.linear_keys = nn.Linear(model_dim,
+ head_count * self.dim_per_head)
+ self.linear_values = nn.Linear(model_dim,
+ head_count * self.dim_per_head)
+ self.linear_query = nn.Linear(model_dim,
+ head_count * self.dim_per_head)
+ self.softmax = nn.Softmax(dim=-1)
+ self.dropout = nn.Dropout(dropout)
+ self.use_final_linear = use_final_linear
+ if (self.use_final_linear):
+ self.final_linear = nn.Linear(model_dim, model_dim)
+
+ # @torchsnooper.snoop()
+ def forward(self, key, value, query, mask=None,
+ layer_cache=None, type=None, predefined_graph_1=None):
+ """
+ Compute the context vector and the attention vectors.
+
+ Args:
+ key (`FloatTensor`): set of `key_len`
+ key vectors `[batch, key_len, dim]`
+ value (`FloatTensor`): set of `key_len`
+ value vectors `[batch, key_len, dim]`
+ query (`FloatTensor`): set of `query_len`
+ query vectors `[batch, query_len, dim]`
+ mask: binary mask indicating which keys have
+ non-zero attention `[batch, query_len, key_len]`
+ Returns:
+ (`FloatTensor`, `FloatTensor`) :
+
+ * output context vectors `[batch, query_len, dim]`
+ * one of the attention vectors `[batch, query_len, key_len]`
+ """
+
+ # CHECKS
+ # batch, k_len, d = key.size()
+ # batch_, k_len_, d_ = value.size()
+ # aeq(batch, batch_)
+ # aeq(k_len, k_len_)
+ # aeq(d, d_)
+ # batch_, q_len, d_ = query.size()
+ # aeq(batch, batch_)
+ # aeq(d, d_)
+ # aeq(self.model_dim % 8, 0)
+ # if mask is not None:
+ # batch_, q_len_, k_len_ = mask.size()
+ # aeq(batch_, batch)
+ # aeq(k_len_, k_len)
+ # aeq(q_len_ == q_len)
+ # END CHECKS
+ batch_size = key.size(0)
+ dim_per_head = self.dim_per_head
+ head_count = self.head_count
+ key_len = key.size(1)
+ query_len = query.size(1)
+
+ def shape(x):
+ """ projection """
+ return x.view(batch_size, -1, head_count, dim_per_head) \
+ .transpose(1, 2)
+
+ def unshape(x):
+ """ compute context """
+ return x.transpose(1, 2).contiguous() \
+ .view(batch_size, -1, head_count * dim_per_head)
+
+ # 1) Project key, value, and query.
+ if layer_cache is not None:
+ if type == "self":
+ query, key, value = self.linear_query(query), \
+ self.linear_keys(query), \
+ self.linear_values(query)
+
+ key = shape(key)
+ value = shape(value)
+
+ if layer_cache is not None:
+ device = key.device
+ if layer_cache["self_keys"] is not None:
+ key = torch.cat(
+ (layer_cache["self_keys"].to(device), key),
+ dim=2)
+ if layer_cache["self_values"] is not None:
+ value = torch.cat(
+ (layer_cache["self_values"].to(device), value),
+ dim=2)
+ layer_cache["self_keys"] = key
+ layer_cache["self_values"] = value
+ elif type == "context":
+ query = self.linear_query(query)
+ if layer_cache is not None:
+ if layer_cache["memory_keys"] is None:
+ key, value = self.linear_keys(key), \
+ self.linear_values(value)
+ key = shape(key)
+ value = shape(value)
+ else:
+ key, value = layer_cache["memory_keys"], \
+ layer_cache["memory_values"]
+ layer_cache["memory_keys"] = key
+ layer_cache["memory_values"] = value
+ else:
+ key, value = self.linear_keys(key), \
+ self.linear_values(value)
+ key = shape(key)
+ value = shape(value)
+ else:
+ key = self.linear_keys(key)
+ value = self.linear_values(value)
+ query = self.linear_query(query)
+ key = shape(key)
+ value = shape(value)
+
+ query = shape(query)
+
+ key_len = key.size(2)
+ query_len = query.size(2)
+
+ # 2) Calculate and scale scores.
+ query = query / math.sqrt(dim_per_head)
+ scores = torch.matmul(query, key.transpose(2, 3))
+
+ if mask is not None:
+ mask = mask.unsqueeze(1).expand_as(scores)
+ scores = scores.masked_fill(mask, -1e18)
+
+ # 3) Apply attention dropout and compute context vectors.
+
+ attn = self.softmax(scores)
+
+ if (not predefined_graph_1 is None):
+ attn_masked = attn[:, -1] * predefined_graph_1
+ attn_masked = attn_masked / (torch.sum(attn_masked, 2).unsqueeze(2) + 1e-9)
+
+ attn = torch.cat([attn[:, :-1], attn_masked.unsqueeze(1)], 1)
+
+ drop_attn = self.dropout(attn)
+ if (self.use_final_linear):
+ context = unshape(torch.matmul(drop_attn, value))
+ output = self.final_linear(context)
+ return output
+ else:
+ context = torch.matmul(drop_attn, value)
+ return context
+
+ # CHECK
+ # batch_, q_len_, d_ = output.size()
+ # aeq(q_len, q_len_)
+ # aeq(batch, batch_)
+ # aeq(d, d_)
+
+ # Return one attn
+
+
+
+class DecoderState(object):
+ """Interface for grouping together the current state of a recurrent
+ decoder. In the simplest case just represents the hidden state of
+ the model. But can also be used for implementing various forms of
+ input_feeding and non-recurrent models.
+
+ Modules need to implement this to utilize beam search decoding.
+ """
+ def detach(self):
+ """ Need to document this """
+ self.hidden = tuple([_.detach() for _ in self.hidden])
+ self.input_feed = self.input_feed.detach()
+
+ def beam_update(self, idx, positions, beam_size):
+ """ Need to document this """
+ for e in self._all:
+ sizes = e.size()
+ br = sizes[1]
+ if len(sizes) == 3:
+ sent_states = e.view(sizes[0], beam_size, br // beam_size,
+ sizes[2])[:, :, idx]
+ else:
+ sent_states = e.view(sizes[0], beam_size,
+ br // beam_size,
+ sizes[2],
+ sizes[3])[:, :, idx]
+
+ sent_states.data.copy_(
+ sent_states.data.index_select(1, positions))
+
+ def map_batch_fn(self, fn):
+ raise NotImplementedError()
+=======
+import math
+
+import torch
+import torch.nn as nn
+import torchsnooper
+
+def aeq(*args):
+ """
+ Assert all arguments have the same value
+ """
+ arguments = (arg for arg in args)
+ first = next(arguments)
+ assert all(arg == first for arg in arguments), \
+ "Not all arguments have the same value: " + str(args)
+
+
+def sequence_mask(lengths, max_len=None):
+ """
+ Creates a boolean mask from sequence lengths.
+ """
+ batch_size = lengths.numel()
+ max_len = max_len or lengths.max()
+ return (torch.arange(0, max_len)
+ .type_as(lengths)
+ .repeat(batch_size, 1)
+ .lt(lengths.unsqueeze(1)))
+
+
+def gelu(x):
+ return 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))
+
+
+""" Global attention modules (Luong / Bahdanau) """
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+class GlobalAttention(nn.Module):
+ """
+ Global attention takes a matrix and a query vector. It
+ then computes a parameterized convex combination of the matrix
+ based on the input query.
+
+ Constructs a unit mapping a query `q` of size `dim`
+ and a source matrix `H` of size `n x dim`, to an output
+ of size `dim`.
+
+
+ .. mermaid::
+
+ graph BT
+ A[Query]
+ subgraph RNN
+ C[H 1]
+ D[H 2]
+ E[H N]
+ end
+ F[Attn]
+ G[Output]
+ A --> F
+ C --> F
+ D --> F
+ E --> F
+ C -.-> G
+ D -.-> G
+ E -.-> G
+ F --> G
+
+ All models compute the output as
+ :math:`c = sum_{j=1}^{SeqLength} a_j H_j` where
+ :math:`a_j` is the softmax of a score function.
+ Then then apply a projection layer to [q, c].
+
+ However they
+ differ on how they compute the attention score.
+
+ * Luong Attention (dot, general):
+ * dot: :math:`score(H_j,q) = H_j^T q`
+ * general: :math:`score(H_j, q) = H_j^T W_a q`
+
+
+ * Bahdanau Attention (mlp):
+ * :math:`score(H_j, q) = v_a^T tanh(W_a q + U_a h_j)`
+
+
+ Args:
+ dim (int): dimensionality of query and key
+ coverage (bool): use coverage term
+ attn_type (str): type of attention to use, options [dot,general,mlp]
+
+ """
+
+ def __init__(self, dim, attn_type="dot"):
+ super(GlobalAttention, self).__init__()
+
+ self.dim = dim
+ assert attn_type in ["dot", "general", "mlp"], (
+ "Please select a valid attention type.")
+ self.attn_type = attn_type
+
+ if self.attn_type == "general":
+ self.linear_in = nn.Linear(dim, dim, bias=False)
+ elif self.attn_type == "mlp":
+ self.linear_context = nn.Linear(dim, dim, bias=False)
+ self.linear_query = nn.Linear(dim, dim, bias=True)
+ self.v = nn.Linear(dim, 1, bias=False)
+ # mlp wants it with bias
+ out_bias = self.attn_type == "mlp"
+ self.linear_out = nn.Linear(dim * 2, dim, bias=out_bias)
+
+
+ def score(self, h_t, h_s):
+ """
+ Args:
+ h_t (`FloatTensor`): sequence of queries `[batch x tgt_len x dim]`
+ h_s (`FloatTensor`): sequence of sources `[batch x src_len x dim]`
+
+ Returns:
+ :obj:`FloatTensor`:
+ raw attention scores (unnormalized) for each src index
+ `[batch x tgt_len x src_len]`
+
+ """
+
+ # Check input sizes
+ src_batch, src_len, src_dim = h_s.size()
+ tgt_batch, tgt_len, tgt_dim = h_t.size()
+
+ if self.attn_type in ["general", "dot"]:
+ if self.attn_type == "general":
+ h_t_ = h_t.view(tgt_batch * tgt_len, tgt_dim)
+ h_t_ = self.linear_in(h_t_)
+ h_t = h_t_.view(tgt_batch, tgt_len, tgt_dim)
+ h_s_ = h_s.transpose(1, 2)
+ # (batch, t_len, d) x (batch, d, s_len) --> (batch, t_len, s_len)
+ return torch.bmm(h_t, h_s_)
+ else:
+ dim = self.dim
+ wq = self.linear_query(h_t.view(-1, dim))
+ wq = wq.view(tgt_batch, tgt_len, 1, dim)
+ wq = wq.expand(tgt_batch, tgt_len, src_len, dim)
+
+ uh = self.linear_context(h_s.contiguous().view(-1, dim))
+ uh = uh.view(src_batch, 1, src_len, dim)
+ uh = uh.expand(src_batch, tgt_len, src_len, dim)
+
+ # (batch, t_len, s_len, d)
+ wquh = torch.tanh(wq + uh)
+
+ return self.v(wquh.view(-1, dim)).view(tgt_batch, tgt_len, src_len)
+
+ def forward(self, source, memory_bank, memory_lengths=None, memory_masks=None):
+ """
+
+ Args:
+ source (`FloatTensor`): query vectors `[batch x tgt_len x dim]`
+ memory_bank (`FloatTensor`): source vectors `[batch x src_len x dim]`
+ memory_lengths (`LongTensor`): the source context lengths `[batch]`
+ coverage (`FloatTensor`): None (not supported yet)
+
+ Returns:
+ (`FloatTensor`, `FloatTensor`):
+
+ * Computed vector `[tgt_len x batch x dim]`
+ * Attention distribtutions for each query
+ `[tgt_len x batch x src_len]`
+ """
+
+ # one step input
+ if source.dim() == 2:
+ one_step = True
+ source = source.unsqueeze(1)
+ else:
+ one_step = False
+
+ batch, source_l, dim = memory_bank.size()
+ batch_, target_l, dim_ = source.size()
+
+ # compute attention scores, as in Luong et al.
+ align = self.score(source, memory_bank)
+
+ if memory_masks is not None:
+ memory_masks = memory_masks.transpose(0,1)
+ memory_masks = memory_masks.transpose(1,2)
+ align.masked_fill_(1 - memory_masks.byte(), -float('inf'))
+
+ if memory_lengths is not None:
+ mask = sequence_mask(memory_lengths, max_len=align.size(-1))
+ mask = mask.unsqueeze(1) # Make it broadcastable.
+ align.masked_fill_(1 - mask, -float('inf'))
+
+ align_vectors = F.softmax(align.view(batch*target_l, source_l), -1)
+ align_vectors = align_vectors.view(batch, target_l, source_l)
+
+ c = torch.bmm(align_vectors, memory_bank)
+
+ # concatenate
+ concat_c = torch.cat([c, source], 2).view(batch*target_l, dim*2)
+ attn_h = self.linear_out(concat_c).view(batch, target_l, dim)
+ if self.attn_type in ["general", "dot"]:
+ attn_h = torch.tanh(attn_h)
+
+ if one_step:
+ attn_h = attn_h.squeeze(1)
+ align_vectors = align_vectors.squeeze(1)
+
+
+ else:
+ attn_h = attn_h.transpose(0, 1).contiguous()
+ align_vectors = align_vectors.transpose(0, 1).contiguous()
+
+ return attn_h, align_vectors
+
+
+class PositionwiseFeedForward(nn.Module):
+ """ A two-layer Feed-Forward-Network with residual layer norm.
+
+ Args:
+ d_model (int): the size of input for the first-layer of the FFN.
+ d_ff (int): the hidden layer size of the second-layer
+ of the FNN.
+ dropout (float): dropout probability in :math:`[0, 1)`.
+ """
+
+ def __init__(self, d_model, d_ff, dropout=0.1):
+ super(PositionwiseFeedForward, self).__init__()
+ self.w_1 = nn.Linear(d_model, d_ff)
+ self.w_2 = nn.Linear(d_ff, d_model)
+ self.layer_norm = nn.LayerNorm(d_model, eps=1e-6)
+ self.actv = gelu
+ self.dropout_1 = nn.Dropout(dropout)
+ self.dropout_2 = nn.Dropout(dropout)
+
+ def forward(self, x):
+ inter = self.dropout_1(self.actv(self.w_1(self.layer_norm(x))))
+ output = self.dropout_2(self.w_2(inter))
+ return output + x
+
+
+class MultiHeadedAttention(nn.Module):
+ """
+ Multi-Head Attention module from
+ "Attention is All You Need"
+ :cite:`DBLP:journals/corr/VaswaniSPUJGKP17`.
+
+ Similar to standard `dot` attention but uses
+ multiple attention distributions simulataneously
+ to select relevant items.
+
+ .. mermaid::
+
+ graph BT
+ A[key]
+ B[value]
+ C[query]
+ O[output]
+ subgraph Attn
+ D[Attn 1]
+ E[Attn 2]
+ F[Attn N]
+ end
+ A --> D
+ C --> D
+ A --> E
+ C --> E
+ A --> F
+ C --> F
+ D --> O
+ E --> O
+ F --> O
+ B --> O
+
+ Also includes several additional tricks.
+
+ Args:
+ head_count (int): number of parallel heads
+ model_dim (int): the dimension of keys/values/queries,
+ must be divisible by head_count
+ dropout (float): dropout parameter
+ """
+
+ def __init__(self, head_count, model_dim, dropout=0.1, use_final_linear=True):
+ assert model_dim % head_count == 0
+ self.dim_per_head = model_dim // head_count
+ self.model_dim = model_dim
+
+ super(MultiHeadedAttention, self).__init__()
+ self.head_count = head_count
+
+ self.linear_keys = nn.Linear(model_dim,
+ head_count * self.dim_per_head)
+ self.linear_values = nn.Linear(model_dim,
+ head_count * self.dim_per_head)
+ self.linear_query = nn.Linear(model_dim,
+ head_count * self.dim_per_head)
+ self.softmax = nn.Softmax(dim=-1)
+ self.dropout = nn.Dropout(dropout)
+ self.use_final_linear = use_final_linear
+ if (self.use_final_linear):
+ self.final_linear = nn.Linear(model_dim, model_dim)
+
+ # @torchsnooper.snoop()
+ def forward(self, key, value, query, mask=None,
+ layer_cache=None, type=None, predefined_graph_1=None):
+ """
+ Compute the context vector and the attention vectors.
+
+ Args:
+ key (`FloatTensor`): set of `key_len`
+ key vectors `[batch, key_len, dim]`
+ value (`FloatTensor`): set of `key_len`
+ value vectors `[batch, key_len, dim]`
+ query (`FloatTensor`): set of `query_len`
+ query vectors `[batch, query_len, dim]`
+ mask: binary mask indicating which keys have
+ non-zero attention `[batch, query_len, key_len]`
+ Returns:
+ (`FloatTensor`, `FloatTensor`) :
+
+ * output context vectors `[batch, query_len, dim]`
+ * one of the attention vectors `[batch, query_len, key_len]`
+ """
+
+ # CHECKS
+ # batch, k_len, d = key.size()
+ # batch_, k_len_, d_ = value.size()
+ # aeq(batch, batch_)
+ # aeq(k_len, k_len_)
+ # aeq(d, d_)
+ # batch_, q_len, d_ = query.size()
+ # aeq(batch, batch_)
+ # aeq(d, d_)
+ # aeq(self.model_dim % 8, 0)
+ # if mask is not None:
+ # batch_, q_len_, k_len_ = mask.size()
+ # aeq(batch_, batch)
+ # aeq(k_len_, k_len)
+ # aeq(q_len_ == q_len)
+ # END CHECKS
+ batch_size = key.size(0)
+ dim_per_head = self.dim_per_head
+ head_count = self.head_count
+ key_len = key.size(1)
+ query_len = query.size(1)
+
+ def shape(x):
+ """ projection """
+ return x.view(batch_size, -1, head_count, dim_per_head) \
+ .transpose(1, 2)
+
+ def unshape(x):
+ """ compute context """
+ return x.transpose(1, 2).contiguous() \
+ .view(batch_size, -1, head_count * dim_per_head)
+
+ # 1) Project key, value, and query.
+ if layer_cache is not None:
+ if type == "self":
+ query, key, value = self.linear_query(query), \
+ self.linear_keys(query), \
+ self.linear_values(query)
+
+ key = shape(key)
+ value = shape(value)
+
+ if layer_cache is not None:
+ device = key.device
+ if layer_cache["self_keys"] is not None:
+ key = torch.cat(
+ (layer_cache["self_keys"].to(device), key),
+ dim=2)
+ if layer_cache["self_values"] is not None:
+ value = torch.cat(
+ (layer_cache["self_values"].to(device), value),
+ dim=2)
+ layer_cache["self_keys"] = key
+ layer_cache["self_values"] = value
+ elif type == "context":
+ query = self.linear_query(query)
+ if layer_cache is not None:
+ if layer_cache["memory_keys"] is None:
+ key, value = self.linear_keys(key), \
+ self.linear_values(value)
+ key = shape(key)
+ value = shape(value)
+ else:
+ key, value = layer_cache["memory_keys"], \
+ layer_cache["memory_values"]
+ layer_cache["memory_keys"] = key
+ layer_cache["memory_values"] = value
+ else:
+ key, value = self.linear_keys(key), \
+ self.linear_values(value)
+ key = shape(key)
+ value = shape(value)
+ else:
+ key = self.linear_keys(key)
+ value = self.linear_values(value)
+ query = self.linear_query(query)
+ key = shape(key)
+ value = shape(value)
+
+ query = shape(query)
+
+ key_len = key.size(2)
+ query_len = query.size(2)
+
+ # 2) Calculate and scale scores.
+ query = query / math.sqrt(dim_per_head)
+ scores = torch.matmul(query, key.transpose(2, 3))
+
+ if mask is not None:
+ mask = mask.unsqueeze(1).expand_as(scores)
+ scores = scores.masked_fill(mask, -1e18)
+
+ # 3) Apply attention dropout and compute context vectors.
+
+ attn = self.softmax(scores)
+
+ if (not predefined_graph_1 is None):
+ attn_masked = attn[:, -1] * predefined_graph_1
+ attn_masked = attn_masked / (torch.sum(attn_masked, 2).unsqueeze(2) + 1e-9)
+
+ attn = torch.cat([attn[:, :-1], attn_masked.unsqueeze(1)], 1)
+
+ drop_attn = self.dropout(attn)
+ if (self.use_final_linear):
+ context = unshape(torch.matmul(drop_attn, value))
+ output = self.final_linear(context)
+ return output
+ else:
+ context = torch.matmul(drop_attn, value)
+ return context
+
+ # CHECK
+ # batch_, q_len_, d_ = output.size()
+ # aeq(q_len, q_len_)
+ # aeq(batch, batch_)
+ # aeq(d, d_)
+
+ # Return one attn
+
+
+
+class DecoderState(object):
+ """Interface for grouping together the current state of a recurrent
+ decoder. In the simplest case just represents the hidden state of
+ the model. But can also be used for implementing various forms of
+ input_feeding and non-recurrent models.
+
+ Modules need to implement this to utilize beam search decoding.
+ """
+ def detach(self):
+ """ Need to document this """
+ self.hidden = tuple([_.detach() for _ in self.hidden])
+ self.input_feed = self.input_feed.detach()
+
+ def beam_update(self, idx, positions, beam_size):
+ """ Need to document this """
+ for e in self._all:
+ sizes = e.size()
+ br = sizes[1]
+ if len(sizes) == 3:
+ sent_states = e.view(sizes[0], beam_size, br // beam_size,
+ sizes[2])[:, :, idx]
+ else:
+ sent_states = e.view(sizes[0], beam_size,
+ br // beam_size,
+ sizes[2],
+ sizes[3])[:, :, idx]
+
+ sent_states.data.copy_(
+ sent_states.data.index_select(1, positions))
+
+ def map_batch_fn(self, fn):
+ raise NotImplementedError()
+>>>>>>> bcb618ae9faf5d35f3147b05b34268a73808238a
diff --git a/fastSum/PreSum/models/optimizers.py b/fastSum/PreSum/models/optimizers.py
new file mode 100644
index 0000000000000000000000000000000000000000..9ee237d9733e3207a7dc488923883607af96c652
--- /dev/null
+++ b/fastSum/PreSum/models/optimizers.py
@@ -0,0 +1,221 @@
+import torch
+from torch import optim
+from torch.nn.utils import clip_grad_norm_
+
+
+def build_optim(args, model, checkpoint):
+ if args.sep_optim:
+ return [build_optim_bert(args, model, checkpoint), build_optim_dec(args, model, checkpoint)]
+ else:
+ return [single_optim(args, model, checkpoint)]
+
+
+def single_optim(args, model, checkpoint):
+ """ Build optimizer """
+
+ if checkpoint is not None:
+ optim = checkpoint['optims'][0]
+ saved_optimizer_state_dict = optim.optimizer.state_dict()
+ optim.optimizer.load_state_dict(saved_optimizer_state_dict)
+ if args.visible_gpus != '-1':
+ for state in optim.optimizer.state.values():
+ for k, v in state.items():
+ if torch.is_tensor(v):
+ state[k] = v.cuda()
+
+ if (optim.method == 'adam') and (len(optim.optimizer.state) < 1):
+ raise RuntimeError(
+ "Error: loaded Adam optimizer from existing model" +
+ " but optimizer state is empty")
+
+ else:
+ optim = Optimizer(
+ args.optim, args.lr, args.max_grad_norm,
+ beta1=args.beta1, beta2=args.beta2,
+ decay_method='noam',
+ warmup_steps=args.warmup_steps)
+
+ optim.set_parameters(list(model.named_parameters()))
+
+ return optim
+
+
+def build_optim_bert(args, model, checkpoint):
+ """ Build optimizer """
+
+ if checkpoint is not None:
+ optim = checkpoint['optims'][0]
+ saved_optimizer_state_dict = optim.optimizer.state_dict()
+ optim.optimizer.load_state_dict(saved_optimizer_state_dict)
+ if args.visible_gpus != '-1':
+ for state in optim.optimizer.state.values():
+ for k, v in state.items():
+ if torch.is_tensor(v):
+ state[k] = v.cuda()
+
+ if (optim.method == 'adam') and (len(optim.optimizer.state) < 1):
+ raise RuntimeError(
+ "Error: loaded Adam optimizer from existing model" +
+ " but optimizer state is empty")
+
+ else:
+ optim = Optimizer(
+ args.optim, args.lr_bert, args.max_grad_norm,
+ beta1=args.beta1, beta2=args.beta2,
+ decay_method='noam',
+ warmup_steps=args.warmup_steps_bert)
+
+ params = [(n, p) for n, p in list(model.named_parameters()) if n.startswith('bert.model')]
+ optim.set_parameters(params)
+
+ return optim
+
+
+def build_optim_dec(args, model, checkpoint):
+ """ Build optimizer """
+
+ if checkpoint is not None:
+ optim = checkpoint['optims'][1]
+ saved_optimizer_state_dict = optim.optimizer.state_dict()
+ optim.optimizer.load_state_dict(saved_optimizer_state_dict)
+ if args.visible_gpus != '-1':
+ for state in optim.optimizer.state.values():
+ for k, v in state.items():
+ if torch.is_tensor(v):
+ state[k] = v.cuda()
+
+ if (optim.method == 'adam') and (len(optim.optimizer.state) < 1):
+ raise RuntimeError(
+ "Error: loaded Adam optimizer from existing model" +
+ " but optimizer state is empty")
+
+ else:
+ optim = Optimizer(
+ args.optim, args.lr_dec, args.max_grad_norm,
+ beta1=args.beta1, beta2=args.beta2,
+ decay_method='noam',
+ warmup_steps=args.warmup_steps_dec)
+
+ params = [(n, p) for n, p in list(model.named_parameters()) if not n.startswith('bert.model')]
+ optim.set_parameters(params)
+
+ return optim
+
+
+class Optimizer(object):
+ """
+ Controller class for optimization. Mostly a thin
+ wrapper for `optim`, but also useful for implementing
+ rate scheduling beyond what is currently available.
+ Also implements necessary methods for training RNNs such
+ as grad manipulations.
+
+ Args:
+ method (:obj:`str`): one of [sgd, adagrad, adadelta, adam]
+ lr (float): learning rate
+ lr_decay (float, optional): learning rate decay multiplier
+ start_decay_steps (int, optional): step to start learning rate decay
+ beta1, beta2 (float, optional): parameters for adam
+ adagrad_accum (float, optional): initialization parameter for adagrad
+ decay_method (str, option): custom decay options
+ warmup_steps (int, option): parameter for `noam` decay
+ model_size (int, option): parameter for `noam` decay
+
+ We use the default parameters for Adam that are suggested by
+ the original paper https://arxiv.org/pdf/1412.6980.pdf
+ These values are also used by other established implementations,
+ e.g. https://www.tensorflow.org/api_docs/python/tf/train/AdamOptimizer
+ https://keras.io/optimizers/
+ Recently there are slightly different values used in the paper
+ "Attention is all you need"
+ https://arxiv.org/pdf/1706.03762.pdf, particularly the value beta2=0.98
+ was used there however, beta2=0.999 is still arguably the more
+ established value, so we use that here as well
+ """
+
+ def __init__(self, method, learning_rate, max_grad_norm,
+ lr_decay=1, start_decay_steps=None, decay_steps=None,
+ beta1=0.9, beta2=0.999,
+ adagrad_accum=0.0,
+ decay_method=None,
+ warmup_steps=4000, weight_decay=0):
+ self.last_ppl = None
+ self.learning_rate = learning_rate
+ self.original_lr = learning_rate
+ self.max_grad_norm = max_grad_norm
+ self.method = method
+ self.lr_decay = lr_decay
+ self.start_decay_steps = start_decay_steps
+ self.decay_steps = decay_steps
+ self.start_decay = False
+ self._step = 0
+ self.betas = [beta1, beta2]
+ self.adagrad_accum = adagrad_accum
+ self.decay_method = decay_method
+ self.warmup_steps = warmup_steps
+ self.weight_decay = weight_decay
+
+ def set_parameters(self, params):
+ """ ? """
+ self.params = []
+ self.sparse_params = []
+ for k, p in params:
+ if p.requires_grad:
+ if self.method != 'sparseadam' or "embed" not in k:
+ self.params.append(p)
+ else:
+ self.sparse_params.append(p)
+ if self.method == 'sgd':
+ self.optimizer = optim.SGD(self.params, lr=self.learning_rate)
+ elif self.method == 'adagrad':
+ self.optimizer = optim.Adagrad(self.params, lr=self.learning_rate)
+ for group in self.optimizer.param_groups:
+ for p in group['params']:
+ self.optimizer.state[p]['sum'] = self.optimizer \
+ .state[p]['sum'].fill_(self.adagrad_accum)
+ elif self.method == 'adadelta':
+ self.optimizer = optim.Adadelta(self.params, lr=self.learning_rate)
+ elif self.method == 'adam':
+ self.optimizer = optim.Adam(self.params, lr=self.learning_rate,
+ betas=self.betas, eps=1e-9)
+ else:
+ raise RuntimeError("Invalid optim method: " + self.method)
+
+ def _set_rate(self, learning_rate):
+ self.learning_rate = learning_rate
+ if self.method != 'sparseadam':
+ self.optimizer.param_groups[0]['lr'] = self.learning_rate
+ else:
+ for op in self.optimizer.optimizers:
+ op.param_groups[0]['lr'] = self.learning_rate
+
+ def step(self):
+ """Update the model parameters based on current gradients.
+
+ Optionally, will employ gradient modification or update learning
+ rate.
+ """
+ self._step += 1
+
+ # Decay method used in tensor2tensor.
+ if self.decay_method == "noam":
+ self._set_rate(
+ self.original_lr *
+ min(self._step ** (-0.5),
+ self._step * self.warmup_steps ** (-1.5)))
+
+ else:
+ if ((self.start_decay_steps is not None) and (
+ self._step >= self.start_decay_steps)):
+ self.start_decay = True
+ if self.start_decay:
+ if ((self._step - self.start_decay_steps)
+ % self.decay_steps == 0):
+ self.learning_rate = self.learning_rate * self.lr_decay
+
+ if self.method != 'sparseadam':
+ self.optimizer.param_groups[0]['lr'] = self.learning_rate
+
+ if self.max_grad_norm:
+ clip_grad_norm_(self.params, self.max_grad_norm)
+ self.optimizer.step()
diff --git a/fastSum/PreSum/others/__init__.py b/fastSum/PreSum/others/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/fastSum/PreSum/others/logging.py b/fastSum/PreSum/others/logging.py
new file mode 100644
index 0000000000000000000000000000000000000000..ceaedff0251dd9986176c27b99ec338b1f95adfc
--- /dev/null
+++ b/fastSum/PreSum/others/logging.py
@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+
+import logging
+
+logger = logging.getLogger()
+
+
+def init_logger(log_file=None, log_file_level=logging.NOTSET):
+ log_format = logging.Formatter("[%(asctime)s %(levelname)s] %(message)s")
+ logger = logging.getLogger()
+ logger.setLevel(logging.INFO)
+
+ console_handler = logging.StreamHandler()
+ console_handler.setFormatter(log_format)
+ logger.handlers = [console_handler]
+
+ if log_file and log_file != '':
+ file_handler = logging.FileHandler(log_file)
+ file_handler.setLevel(log_file_level)
+ file_handler.setFormatter(log_format)
+ logger.addHandler(file_handler)
+
+ return logger
diff --git a/fastSum/PreSum/others/tokenization.py b/fastSum/PreSum/others/tokenization.py
new file mode 100644
index 0000000000000000000000000000000000000000..acb9d3e1a2821e042ae1acfbdcb0d44f27af422b
--- /dev/null
+++ b/fastSum/PreSum/others/tokenization.py
@@ -0,0 +1,382 @@
+# coding=utf-8
+# Copyright 2018 The Google AI Language Team Authors and The HugginFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tokenization classes."""
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import collections
+import logging
+import os
+import unicodedata
+from io import open
+
+from pytorch_transformers import cached_path
+
+logger = logging.getLogger(__name__)
+
+PRETRAINED_VOCAB_ARCHIVE_MAP = {
+ 'bert-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt",
+ 'bert-large-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-vocab.txt",
+ 'bert-base-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-vocab.txt",
+ 'bert-large-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-vocab.txt",
+ 'bert-base-multilingual-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-uncased-vocab.txt",
+ 'bert-base-multilingual-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-cased-vocab.txt",
+ 'bert-base-chinese': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-chinese-vocab.txt",
+}
+PRETRAINED_VOCAB_POSITIONAL_EMBEDDINGS_SIZE_MAP = {
+ 'bert-base-uncased': 512,
+ 'bert-large-uncased': 512,
+ 'bert-base-cased': 512,
+ 'bert-large-cased': 512,
+ 'bert-base-multilingual-uncased': 512,
+ 'bert-base-multilingual-cased': 512,
+ 'bert-base-chinese': 512,
+}
+VOCAB_NAME = 'vocab.txt'
+
+
+def load_vocab(vocab_file):
+ """Loads a vocabulary file into a dictionary."""
+ vocab = collections.OrderedDict()
+ index = 0
+ with open(vocab_file, "r", encoding="utf-8") as reader:
+ while True:
+ token = reader.readline()
+ if not token:
+ break
+ token = token.strip()
+ vocab[token] = index
+ index += 1
+ return vocab
+
+
+def whitespace_tokenize(text):
+ """Runs basic whitespace cleaning and splitting on a peice of text."""
+ text = text.strip()
+ if not text:
+ return []
+ tokens = text.split()
+ return tokens
+
+
+class BertTokenizer(object):
+ """Runs end-to-end tokenization: punctuation splitting + wordpiece"""
+
+ def __init__(self, vocab_file, do_lower_case=True, max_len=None,
+ never_split=("[UNK]", "[SEP]", "[PAD]", "[CLS]", "[MASK]", "[unused0]", "[unused1]", "[unused2]", "[unused3]", "[unused4]", "[unused5]", "[unused6]")):
+
+ if not os.path.isfile(vocab_file):
+ raise ValueError(
+ "Can't find a vocabulary file at path '{}'. To load the vocabulary from a Google pretrained "
+ "model use `tokenizer = BertTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`".format(vocab_file))
+ self.do_lower_case = do_lower_case
+ self.vocab = load_vocab(vocab_file)
+ self.ids_to_tokens = collections.OrderedDict(
+ [(ids, tok) for tok, ids in self.vocab.items()])
+ self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case,
+ never_split=never_split)
+ self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab)
+ self.max_len = max_len if max_len is not None else int(1e12)
+
+ def tokenize(self, text, use_bert_basic_tokenizer=False):
+ split_tokens = []
+ if(use_bert_basic_tokenizer):
+ pretokens = self.basic_tokenizer.tokenize(text)
+ else:
+ pretokens = list(enumerate(text.split()))
+
+ for i,token in pretokens:
+ # if(self.do_lower_case):
+ # token = token.lower()
+ subtokens = self.wordpiece_tokenizer.tokenize(token)
+ for sub_token in subtokens:
+ split_tokens.append(sub_token)
+ return split_tokens
+
+ def convert_tokens_to_ids(self, tokens):
+ """Converts a sequence of tokens into ids using the vocab."""
+ ids = []
+ for token in tokens:
+ ids.append(self.vocab[token])
+ # if len(ids) > self.max_len:
+ # raise ValueError(
+ # "Token indices sequence length is longer than the specified maximum "
+ # " sequence length for this BERT model ({} > {}). Running this"
+ # " sequence through BERT will result in indexing errors".format(len(ids), self.max_len)
+ # )
+ return ids
+
+ def convert_ids_to_tokens(self, ids):
+ """Converts a sequence of ids in wordpiece tokens using the vocab."""
+ tokens = []
+ for i in ids:
+ tokens.append(self.ids_to_tokens[i])
+ return tokens
+
+ @classmethod
+ def from_pretrained(cls, pretrained_model_name_or_path, cache_dir=None, *inputs, **kwargs):
+ """
+ Instantiate a PreTrainedBertModel from a pre-trained model file.
+ Download and cache the pre-trained model file if needed.
+ """
+ if pretrained_model_name_or_path in PRETRAINED_VOCAB_ARCHIVE_MAP:
+ vocab_file = PRETRAINED_VOCAB_ARCHIVE_MAP[pretrained_model_name_or_path]
+ else:
+ vocab_file = pretrained_model_name_or_path
+ if os.path.isdir(vocab_file):
+ vocab_file = os.path.join(vocab_file, VOCAB_NAME)
+ # redirect to the cache, if necessary
+ try:
+ resolved_vocab_file = cached_path(vocab_file, cache_dir=cache_dir)
+ except EnvironmentError:
+ logger.error(
+ "Model name '{}' was not found in model name list ({}). "
+ "We assumed '{}' was a path or url but couldn't find any file "
+ "associated to this path or url.".format(
+ pretrained_model_name_or_path,
+ ', '.join(PRETRAINED_VOCAB_ARCHIVE_MAP.keys()),
+ vocab_file))
+ return None
+ if resolved_vocab_file == vocab_file:
+ logger.info("loading vocabulary file {}".format(vocab_file))
+ else:
+ logger.info("loading vocabulary file {} from cache at {}".format(
+ vocab_file, resolved_vocab_file))
+ if pretrained_model_name_or_path in PRETRAINED_VOCAB_POSITIONAL_EMBEDDINGS_SIZE_MAP:
+ # if we're using a pretrained model, ensure the tokenizer wont index sequences longer
+ # than the number of positional embeddings
+ max_len = PRETRAINED_VOCAB_POSITIONAL_EMBEDDINGS_SIZE_MAP[pretrained_model_name_or_path]
+ kwargs['max_len'] = min(kwargs.get('max_len', int(1e12)), max_len)
+ # Instantiate tokenizer.
+ tokenizer = cls(resolved_vocab_file, *inputs, **kwargs)
+ return tokenizer
+
+
+class BasicTokenizer(object):
+ """Runs basic tokenization (punctuation splitting, lower casing, etc.)."""
+
+ def __init__(self,
+ do_lower_case=True,
+ never_split=("[UNK]", "[SEP]", "[PAD]", "[CLS]", "[MASK]")):
+ """Constructs a BasicTokenizer.
+
+ Args:
+ do_lower_case: Whether to lower case the input.
+ """
+ self.do_lower_case = do_lower_case
+ self.never_split = never_split
+
+ def tokenize(self, text):
+ """Tokenizes a piece of text."""
+ text = self._clean_text(text)
+ # This was added on November 1st, 2018 for the multilingual and Chinese
+ # models. This is also applied to the English models now, but it doesn't
+ # matter since the English models were not trained on any Chinese data
+ # and generally don't have any Chinese data in them (there are Chinese
+ # characters in the vocabulary because Wikipedia does have some Chinese
+ # words in the English Wikipedia.).
+ text = self._tokenize_chinese_chars(text)
+ orig_tokens = whitespace_tokenize(text)
+ split_tokens = []
+ for i,token in enumerate(orig_tokens):
+ if self.do_lower_case and token not in self.never_split:
+ token = token.lower()
+ token = self._run_strip_accents(token)
+ # split_tokens.append(token)
+ split_tokens.extend([(i,t) for t in self._run_split_on_punc(token)])
+
+ # output_tokens = whitespace_tokenize(" ".join(split_tokens))
+ return split_tokens
+
+ def _run_strip_accents(self, text):
+ """Strips accents from a piece of text."""
+ text = unicodedata.normalize("NFD", text)
+ output = []
+ for char in text:
+ cat = unicodedata.category(char)
+ if cat == "Mn":
+ continue
+ output.append(char)
+ return "".join(output)
+
+ def _run_split_on_punc(self, text):
+ """Splits punctuation on a piece of text."""
+ if text in self.never_split:
+ return [text]
+ chars = list(text)
+ i = 0
+ start_new_word = True
+ output = []
+ while i < len(chars):
+ char = chars[i]
+ if _is_punctuation(char):
+ output.append([char])
+ start_new_word = True
+ else:
+ if start_new_word:
+ output.append([])
+ start_new_word = False
+ output[-1].append(char)
+ i += 1
+
+ return ["".join(x) for x in output]
+
+ def _tokenize_chinese_chars(self, text):
+ """Adds whitespace around any CJK character."""
+ output = []
+ for char in text:
+ cp = ord(char)
+ if self._is_chinese_char(cp):
+ output.append(" ")
+ output.append(char)
+ output.append(" ")
+ else:
+ output.append(char)
+ return "".join(output)
+
+ def _is_chinese_char(self, cp):
+ """Checks whether CP is the codepoint of a CJK character."""
+ # This defines a "chinese character" as anything in the CJK Unicode block:
+ # https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
+ #
+ # Note that the CJK Unicode block is NOT all Japanese and Korean characters,
+ # despite its name. The modern Korean Hangul alphabet is a different block,
+ # as is Japanese Hiragana and Katakana. Those alphabets are used to write
+ # space-separated words, so they are not treated specially and handled
+ # like the all of the other languages.
+ if ((cp >= 0x4E00 and cp <= 0x9FFF) or #
+ (cp >= 0x3400 and cp <= 0x4DBF) or #
+ (cp >= 0x20000 and cp <= 0x2A6DF) or #
+ (cp >= 0x2A700 and cp <= 0x2B73F) or #
+ (cp >= 0x2B740 and cp <= 0x2B81F) or #
+ (cp >= 0x2B820 and cp <= 0x2CEAF) or
+ (cp >= 0xF900 and cp <= 0xFAFF) or #
+ (cp >= 0x2F800 and cp <= 0x2FA1F)): #
+ return True
+
+ return False
+
+ def _clean_text(self, text):
+ """Performs invalid character removal and whitespace cleanup on text."""
+ output = []
+ for char in text:
+ cp = ord(char)
+ if cp == 0 or cp == 0xfffd or _is_control(char):
+ continue
+ if _is_whitespace(char):
+ output.append(" ")
+ else:
+ output.append(char)
+ return "".join(output)
+
+
+class WordpieceTokenizer(object):
+ """Runs WordPiece tokenization."""
+
+ def __init__(self, vocab, unk_token="[UNK]", max_input_chars_per_word=100):
+ self.vocab = vocab
+ self.unk_token = unk_token
+ self.max_input_chars_per_word = max_input_chars_per_word
+
+ def tokenize(self, text):
+ """Tokenizes a piece of text into its word pieces.
+
+ This uses a greedy longest-match-first algorithm to perform tokenization
+ using the given vocabulary.
+
+ For example:
+ input = "unaffable"
+ output = ["un", "##aff", "##able"]
+
+ Args:
+ text: A single token or whitespace separated tokens. This should have
+ already been passed through `BasicTokenizer`.
+
+ Returns:
+ A list of wordpiece tokens.
+ """
+
+ output_tokens = []
+ for token in whitespace_tokenize(text):
+ chars = list(token)
+ if len(chars) > self.max_input_chars_per_word:
+ output_tokens.append(self.unk_token)
+ continue
+
+ is_bad = False
+ start = 0
+ sub_tokens = []
+ while start < len(chars):
+ end = len(chars)
+ cur_substr = None
+ while start < end:
+ substr = "".join(chars[start:end])
+ if start > 0:
+ substr = "##" + substr
+ if substr in self.vocab:
+ cur_substr = substr
+ break
+ end -= 1
+ if cur_substr is None:
+ is_bad = True
+ break
+ sub_tokens.append(cur_substr)
+ start = end
+
+ if is_bad:
+ output_tokens.append(self.unk_token)
+ else:
+ output_tokens.extend(sub_tokens)
+ return output_tokens
+
+
+def _is_whitespace(char):
+ """Checks whether `chars` is a whitespace character."""
+ # \t, \n, and \r are technically contorl characters but we treat them
+ # as whitespace since they are generally considered as such.
+ if char == " " or char == "\t" or char == "\n" or char == "\r":
+ return True
+ cat = unicodedata.category(char)
+ if cat == "Zs":
+ return True
+ return False
+
+
+def _is_control(char):
+ """Checks whether `chars` is a control character."""
+ # These are technically control characters but we count them as whitespace
+ # characters.
+ if char == "\t" or char == "\n" or char == "\r":
+ return False
+ cat = unicodedata.category(char)
+ if cat.startswith("C"):
+ return True
+ return False
+
+
+def _is_punctuation(char):
+ """Checks whether `chars` is a punctuation character."""
+ cp = ord(char)
+ # We treat all non-letter/number ASCII as punctuation.
+ # Characters such as "^", "$", and "`" are not in the Unicode
+ # Punctuation class but we treat them as punctuation anyways, for
+ # consistency.
+ if ((cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or
+ (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126)):
+ return True
+ cat = unicodedata.category(char)
+ if cat.startswith("P"):
+ return True
+ return False
diff --git a/fastSum/PreSum/others/utils.py b/fastSum/PreSum/others/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..31df5354c9a5509a5e12ddbe975524d501e33daf
--- /dev/null
+++ b/fastSum/PreSum/others/utils.py
@@ -0,0 +1,419 @@
+import os
+import re
+import shutil
+import time
+
+# from others import pyrouge
+# import pyrouge
+import argparse
+
+REMAP = {"-lrb-": "(", "-rrb-": ")", "-lcb-": "{", "-rcb-": "}",
+ "-lsb-": "[", "-rsb-": "]", "``": '"', "''": '"'}
+
+
+def clean(x):
+ return re.sub(
+ r"-lrb-|-rrb-|-lcb-|-rcb-|-lsb-|-rsb-|``|''",
+ lambda m: REMAP.get(m.group()), x)
+
+ROOT = "../"
+_ROUGE_PATH = '../../resources/ROUGE/RELEASE-1.5.5'
+
+def mkdir(path):
+ if not os.path.exists(path):
+ os.mkdir(path)
+
+
+def str2bool(v):
+ if v.lower() in ('yes', 'true', 't', 'y', '1'):
+ return True
+ elif v.lower() in ('no', 'false', 'f', 'n', '0'):
+ return False
+ else:
+ raise argparse.ArgumentTypeError('Boolean value expected.')
+
+
+def _get_ngrams(n, text):
+ """Calcualtes n-grams.
+
+ Args:
+ n: which n-grams to calculate
+ text: An array of tokens
+
+ Returns:
+ A set of n-grams
+ """
+ ngram_set = set()
+ text_length = len(text)
+ max_index_ngram_start = text_length - n
+ for i in range(max_index_ngram_start + 1):
+ ngram_set.add(tuple(text[i:i + n]))
+ return ngram_set
+
+
+def _get_word_ngrams(n, sentences):
+ """Calculates word n-grams for multiple sentences.
+ """
+ assert len(sentences) > 0
+ assert n > 0
+
+ # words = _split_into_words(sentences)
+
+ words = sum(sentences, [])
+ # words = [w for w in words if w not in stopwords]
+ return _get_ngrams(n, words)
+
+
+def process(params):
+ temp_dir, data = params
+ candidates, references, pool_id = data
+ cnt = len(candidates)
+ current_time = time.strftime('%Y-%m-%d-%H-%M-%S', time.localtime())
+ tmp_dir = os.path.join(temp_dir, "rouge-tmp-{}-{}".format(current_time, pool_id))
+ if not os.path.isdir(tmp_dir):
+ os.mkdir(tmp_dir)
+ os.mkdir(tmp_dir + "/candidate")
+ os.mkdir(tmp_dir + "/reference")
+ try:
+
+ for i in range(cnt):
+ if len(references[i]) < 1:
+ continue
+ with open(tmp_dir + "/candidate/cand.{}.txt".format(i), "w",
+ encoding="utf-8") as f:
+ f.write(candidates[i])
+ with open(tmp_dir + "/reference/ref.{}.txt".format(i), "w",
+ encoding="utf-8") as f:
+ f.write(references[i])
+ r = pyrouge.Rouge155(temp_dir=temp_dir)
+ r.model_dir = tmp_dir + "/reference/"
+ r.system_dir = tmp_dir + "/candidate/"
+ r.model_filename_pattern = 'ref.#ID#.txt'
+ r.system_filename_pattern = r'cand.(\d+).txt'
+ rouge_results = r.convert_and_evaluate()
+ print(rouge_results)
+ results_dict = r.output_to_dict(rouge_results)
+ finally:
+ pass
+ if os.path.isdir(tmp_dir):
+ shutil.rmtree(tmp_dir)
+ return results_dict
+
+
+def test_rouge(temp_dir, cand, ref):
+ candidates = [line.strip() for line in open(cand, encoding='utf-8')]
+ references = [line.strip() for line in open(ref, encoding='utf-8')]
+ print(len(candidates))
+ print(len(references))
+ assert len(candidates) == len(references)
+
+ cnt = len(candidates)
+ current_time = time.strftime('%Y-%m-%d-%H-%M-%S', time.localtime())
+ tmp_dir = os.path.join(temp_dir, "rouge-tmp-{}".format(current_time))
+ if not os.path.isdir(tmp_dir):
+ os.mkdir(tmp_dir)
+ os.mkdir(tmp_dir + "/candidate")
+ os.mkdir(tmp_dir + "/reference")
+ try:
+
+ for i in range(cnt):
+ if len(references[i]) < 1:
+ continue
+ with open(tmp_dir + "/candidate/cand.{}.txt".format(i), "w",
+ encoding="utf-8") as f:
+ f.write(candidates[i])
+ with open(tmp_dir + "/reference/ref.{}.txt".format(i), "w",
+ encoding="utf-8") as f:
+ f.write(references[i])
+ r = pyrouge.Rouge155(temp_dir=temp_dir)
+ r.model_dir = tmp_dir + "/reference/"
+ r.system_dir = tmp_dir + "/candidate/"
+ r.model_filename_pattern = 'ref.#ID#.txt'
+ r.system_filename_pattern = r'cand.(\d+).txt'
+ rouge_results = r.convert_and_evaluate()
+ print(rouge_results)
+ results_dict = r.output_to_dict(rouge_results)
+ finally:
+ pass
+ if os.path.isdir(tmp_dir):
+ shutil.rmtree(tmp_dir)
+ return results_dict
+
+
+def tile(x, count, dim=0):
+ """
+ Tiles x on dimension dim count times.
+ """
+ perm = list(range(len(x.size())))
+ if dim != 0:
+ perm[0], perm[dim] = perm[dim], perm[0]
+ x = x.permute(perm).contiguous()
+ out_size = list(x.size())
+ out_size[0] *= count
+ batch = x.size(0)
+ x = x.view(batch, -1) \
+ .transpose(0, 1) \
+ .repeat(count, 1) \
+ .transpose(0, 1) \
+ .contiguous() \
+ .view(*out_size)
+ if dim != 0:
+ x = x.permute(perm).contiguous()
+ return x
+
+
+def rouge_results_to_str(results_dict):
+ return ">> ROUGE-F(1/2/3/l): {:.2f}/{:.2f}/{:.2f}\nROUGE-R(1/2/3/l): {:.2f}/{:.2f}/{:.2f}\n".format(
+ results_dict["rouge_1_f_score"] * 100,
+ results_dict["rouge_2_f_score"] * 100,
+ # results_dict["rouge_3_f_score"] * 100,
+ results_dict["rouge_l_f_score"] * 100,
+ results_dict["rouge_1_recall"] * 100,
+ results_dict["rouge_2_recall"] * 100,
+ # results_dict["rouge_3_f_score"] * 100,
+ results_dict["rouge_l_recall"] * 100
+
+ # ,results_dict["rouge_su*_f_score"] * 100
+ )
+
+
+import os
+from os.path import exists
+
+
+def get_data_path(mode, label_type):
+ '''
+ :param mode: train, validate or test
+ :param label_type: dataset dir name
+ :return:
+ '''
+ paths = {}
+ if mode == 'train':
+ paths['train'] = 'data/' + label_type + '/bert.train.jsonl'
+ paths['val'] = 'data/' + label_type + '/bert.val.jsonl'
+ else:
+ paths['test'] = 'data/' + label_type + '/bert.test.jsonl'
+ return paths
+
+
+def get_rouge_path(label_type):
+ # if label_type == 'others':
+ # data_path = 'data/' + label_type + '/bert.test.jsonl'
+ # else:
+ # data_path = 'data/' + label_type + '/test.jsonl'
+ data_path = 'data/' + label_type + '/bert.test.jsonl'
+ dec_path = 'dec'
+ ref_path = 'ref'
+ if not exists(ref_path):
+ os.makedirs(ref_path)
+ if not exists(dec_path):
+ os.makedirs(dec_path)
+ return data_path, dec_path, ref_path
+
+
+def configure_training(args):
+ devices = [int(gpu) for gpu in args.visible_gpus.split(',')]
+ params = {}
+ params['task'] = args.task
+ params['encoder'] = args.encoder
+ params['mode'] = args.mode
+ params['max_pos'] = args.max_pos
+ params['sep_optim'] = args.sep_optim
+ params['beam_size'] = args.beam_size
+ params['batch_size'] = args.batch_size
+ params['accum_count'] = args.accum_count
+ params['lr_bert'] = args.lr_bert
+ params['lr_dec'] = args.lr_dec
+ params['lr'] = args.lr
+ params['warmup_steps'] = args.warmup_steps
+ params['warmup_steps_bert'] = args.warmup_steps_bert
+ params['warmup_steps_dec'] = args.warmup_steps_dec
+ params['n_epochs'] = args.n_epochs
+ # params['valid_steps'] = args.valid_steps
+ return devices, params
+
+
+import re
+import os
+import shutil
+import copy
+import datetime
+import numpy as np
+from rouge import Rouge
+import random
+import tempfile
+import os
+import pyrouge
+import logging
+import tensorflow as tf
+import datetime
+
+import sys
+
+sys.setrecursionlimit(10000)
+
+REMAP = {"-lrb-": "(", "-rrb-": ")", "-lcb-": "{", "-rcb-": "}",
+ "-lsb-": "[", "-rsb-": "]", "``": '"', "''": '"'}
+
+
+def ranstr(num):
+ # 猜猜变量名为啥叫 H,生成随机字符串
+ H = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'
+
+ salt = ''
+ for i in range(num):
+ salt += random.choice(H)
+
+ return salt
+
+
+def clean(x):
+ return re.sub(r"-lrb-|-rrb-|-lcb-|-rcb-|-lsb-|-rsb-|``|''", lambda m: REMAP.get(m.group()), x)
+
+
+def pyrouge_score_all(hyps_list, refer_list, config, tmp_path, remap=True):
+ nowTime = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
+
+ tempfile.tempdir = tmp_path
+ assert not os.path.exists(tmp_path)
+ os.mkdir(tmp_path)
+
+ PYROUGE_ROOT = os.path.join(ROOT, 'tmp', nowTime + "_" + ranstr(6))
+ while os.path.exists(PYROUGE_ROOT):
+ PYROUGE_ROOT = os.path.join(ROOT, 'tmp', nowTime + "_" + ranstr(6))
+ assert not os.path.exists(PYROUGE_ROOT), "pyrouge root already exist!"
+
+ SYSTEM_PATH = os.path.join(PYROUGE_ROOT, 'model')
+ MODEL_PATH = os.path.join(PYROUGE_ROOT, 'gold')
+
+ os.makedirs(SYSTEM_PATH)
+ os.makedirs(MODEL_PATH)
+
+ assert len(hyps_list) == len(refer_list)
+ gold_path = os.path.join(config.decode_path, "gold.txt")
+ pred_path = os.path.join(config.decode_path, "pred.txt")
+
+ for i in range(len(hyps_list)):
+ system_file = os.path.join(SYSTEM_PATH, 'Model.%d.txt' % i)
+ model_file = os.path.join(MODEL_PATH, 'Gold.A.%d.txt' % i)
+
+ refer = clean(refer_list[i]) if remap else refer_list[i]
+ hyps = clean(hyps_list[i]) if remap else hyps_list[i]
+
+ with open(system_file, 'wb') as f:
+ f.write(hyps.encode('utf-8'))
+ with open(model_file, 'wb') as f:
+ f.write(refer.encode('utf-8'))
+
+ with open(gold_path, 'a') as f:
+ f.write(refer.replace("\n", " "))
+ f.write("\n")
+ with open(pred_path, 'a') as f:
+ f.write(hyps.replace("\n", " "))
+ f.write("\n")
+
+ r = pyrouge.Rouge155(_ROUGE_PATH)
+
+ r.system_dir = SYSTEM_PATH
+ r.model_dir = MODEL_PATH
+ r.system_filename_pattern = 'Model.(\d+).txt'
+ r.model_filename_pattern = 'Gold.[A-Z].#ID#.txt'
+
+ output = r.convert_and_evaluate(rouge_args="-e {}/data -a -m -n 2 -d".format(_ROUGE_PATH))
+ output_dict = r.output_to_dict(output)
+
+ shutil.rmtree(PYROUGE_ROOT)
+ shutil.rmtree(tmp_path)
+
+ scores = {}
+ scores['rouge_1_precision'], scores['rouge_1_recall'], scores['rouge_1_f_score'] = output_dict['rouge_1_precision'], \
+ output_dict['rouge_1_recall'], \
+ output_dict[
+ 'rouge_1_f_score']
+ scores['rouge_2_precision'], scores['rouge_2_recall'], scores['rouge_2_f_score'] = output_dict['rouge_2_precision'], \
+ output_dict['rouge_2_recall'], \
+ output_dict[
+ 'rouge_2_f_score']
+ scores['rouge_l_precision'], scores['rouge_l_recall'], scores['rouge_l_f_score'] = output_dict['rouge_l_precision'], \
+ output_dict['rouge_l_recall'], \
+ output_dict[
+ 'rouge_l_f_score']
+ return scores
+
+
+def pyrouge_score_all_multi(hyps_list, refer_list, config, tmp_path, remap=True):
+ # 暂时不能用，使用时需要检查过
+ nowTime = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
+
+ tempfile.tempdir = tmp_path
+ assert not os.path.exists(tmp_path)
+ os.mkdir(tmp_path)
+
+ PYROUGE_ROOT = os.path.join(ROOT, 'tmp', nowTime + "_" + ranstr(6))
+ while os.path.exists(PYROUGE_ROOT):
+ PYROUGE_ROOT = os.path.join(ROOT, 'tmp', nowTime + "_" + ranstr(6))
+ assert not os.path.exists(PYROUGE_ROOT), "pyrouge root already exist!"
+
+ SYSTEM_PATH = os.path.join(PYROUGE_ROOT, 'model')
+ MODEL_PATH = os.path.join(PYROUGE_ROOT, 'gold')
+
+ os.makedirs(SYSTEM_PATH)
+ os.makedirs(MODEL_PATH)
+
+ assert len(hyps_list) == len(refer_list)
+
+ pred_path = os.path.join(config.decode_path, "pred.txt")
+
+ for i in range(len(hyps_list)):
+ system_file = os.path.join(SYSTEM_PATH, 'Model.%d.txt' % i)
+ # model_file = os.path.join(MODEL_PATH, 'Reference.A.%d.txt' % i)
+
+ hyps = clean(hyps_list[i]) if remap else hyps_list[i]
+
+ with open(system_file, 'wb') as f:
+ f.write(hyps.encode('utf-8'))
+
+ referType = ["A", "B", "C", "D", "E", "F", "G"]
+
+ for j in range(len(refer_list[i])):
+ model_file = os.path.join(MODEL_PATH, "Gold.%s.%d.txt" % (referType[j], i))
+ refer = clean(refer_list[i][j]) if remap else refer_list[i][j]
+ with open(model_file, 'wb') as f:
+ f.write(refer.encode('utf-8'))
+
+ gold_path = os.path.join(config.decode_path, "gold.%s.txt" % (referType[j]))
+ with open(gold_path, 'a') as f:
+ f.write(refer.replace("\n", " "))
+ f.write("\n")
+
+ with open(pred_path, 'a') as f:
+ f.write(hyps.replace("\n", " "))
+ f.write("\n")
+
+ r = pyrouge.Rouge155()
+
+ r.system_dir = SYSTEM_PATH
+ r.model_dir = MODEL_PATH
+ r.system_filename_pattern = 'Model.(\d+).txt'
+ r.model_filename_pattern = 'Gold.[A-Z].#ID#.txt'
+
+ output = r.convert_and_evaluate()
+ output_dict = r.output_to_dict(output)
+
+ shutil.rmtree(PYROUGE_ROOT)
+ shutil.rmtree(tmp_path)
+
+ scores = {}
+ scores["rouge_1_precision"], scores['rouge_1_recall'], scores['rouge_1_f_score'] = output_dict['rouge_1_precision'], \
+ output_dict['rouge_1_recall'], \
+ output_dict[
+ 'rouge_1_f_score']
+ scores['rouge_2_precision'], scores['rouge_2_recall'], scores['rouge_2_f_score'] = output_dict['rouge_2_precision'], \
+ output_dict['rouge_2_recall'], \
+ output_dict[
+ 'rouge_2_f_score']
+ scores['rouge_l_precision'], scores['rouge_l_recall'], scores['rouge_l_f_score'] = output_dict['rouge_l_precision'], \
+ output_dict['rouge_l_recall'], \
+ output_dict[
+ 'rouge_l_f_score']
+ return scores
diff --git a/fastSum/PreSum/preprocess.py b/fastSum/PreSum/preprocess.py
new file mode 100644
index 0000000000000000000000000000000000000000..094ec7af0da9bf4741ec7622e73c48cd24cc91f1
--- /dev/null
+++ b/fastSum/PreSum/preprocess.py
@@ -0,0 +1,260 @@
+# encoding=utf-8
+
+import argparse
+import time
+
+from others.logging import init_logger
+from others.utils import str2bool, _get_word_ngrams, mkdir
+import gc
+import glob
+import hashlib
+import json
+import os
+import re
+import subprocess
+from os.path import join as pjoin
+import torch
+from multiprocess import Pool
+from others.logging import logger
+from others.tokenization import BertTokenizer
+
+
+def load_jsonl(data_path):
+ data = []
+ with open(data_path) as f:
+ for line in f:
+ data.append(json.loads(line))
+ return data
+
+
+def cal_rouge(evaluated_ngrams, reference_ngrams):
+ reference_count = len(reference_ngrams)
+ evaluated_count = len(evaluated_ngrams)
+
+ overlapping_ngrams = evaluated_ngrams.intersection(reference_ngrams)
+ overlapping_count = len(overlapping_ngrams)
+
+ if evaluated_count == 0:
+ precision = 0.0
+ else:
+ precision = overlapping_count / evaluated_count
+
+ if reference_count == 0:
+ recall = 0.0
+ else:
+ recall = overlapping_count / reference_count
+
+ f1_score = 2.0 * ((precision * recall) / (precision + recall + 1e-8))
+ return {"f": f1_score, "p": precision, "r": recall}
+
+
+def greedy_selection(doc_sent_list, abstract_sent_list, summary_size=3):
+ def _rouge_clean(s):
+ return re.sub(r'[^a-zA-Z0-9 ]', '', s)
+
+ max_rouge = 0.0
+ abstract = sum(abstract_sent_list, [])
+ abstract = _rouge_clean(' '.join(abstract)).split()
+ sents = [_rouge_clean(' '.join(s)).split() for s in doc_sent_list]
+ evaluated_1grams = [_get_word_ngrams(1, [sent]) for sent in sents]
+ reference_1grams = _get_word_ngrams(1, [abstract])
+ evaluated_2grams = [_get_word_ngrams(2, [sent]) for sent in sents]
+ reference_2grams = _get_word_ngrams(2, [abstract])
+
+ selected = []
+ # 把for s in range(len(summary_size)) 改成 for s in range(len(abstract_sent_list)) 消除hard code
+ for s in range(len(abstract_sent_list)):
+ cur_max_rouge = max_rouge
+ cur_id = -1
+ for i in range(len(sents)):
+ if (i in selected):
+ continue
+ c = selected + [i]
+ candidates_1 = [evaluated_1grams[idx] for idx in c]
+ candidates_1 = set.union(*map(set, candidates_1))
+ candidates_2 = [evaluated_2grams[idx] for idx in c]
+ candidates_2 = set.union(*map(set, candidates_2))
+ rouge_1 = cal_rouge(candidates_1, reference_1grams)['f']
+ rouge_2 = cal_rouge(candidates_2, reference_2grams)['f']
+ rouge_score = rouge_1 + rouge_2
+ if rouge_score > cur_max_rouge:
+ cur_max_rouge = rouge_score
+ cur_id = i
+ if (cur_id == -1):
+ return selected
+ selected.append(cur_id)
+ max_rouge = cur_max_rouge
+
+ return sorted(selected)
+
+
+def hashhex(s):
+ """Returns a heximal formated SHA1 hash of the input string."""
+ h = hashlib.sha1()
+ h.update(s.encode('utf-8'))
+ return h.hexdigest()
+
+
+class BertData():
+ def __init__(self, args):
+ self.args = args
+ self.tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)
+
+ self.sep_token = '[SEP]'
+ self.cls_token = '[CLS]'
+ self.pad_token = '[PAD]'
+ self.tgt_bos = '[unused0]'
+ self.tgt_eos = '[unused1]'
+ self.tgt_sent_split = '[unused2]'
+ self.sep_vid = self.tokenizer.vocab[self.sep_token]
+ self.cls_vid = self.tokenizer.vocab[self.cls_token]
+ self.pad_vid = self.tokenizer.vocab[self.pad_token]
+
+ def preprocess(self, src, tgt, sent_labels, real_sent_labels, use_bert_basic_tokenizer=False, is_test=False):
+
+ if (not is_test) and len(src) == 0:
+ return None
+
+ original_src_txt = [' '.join(s) for s in src]
+ idxs = [i for i, s in enumerate(src) if (len(s) > self.args.min_src_ntokens_per_sent)]
+
+ _sent_labels = [0] * len(src)
+ for l in sent_labels:
+ _sent_labels[l] = 1
+
+ _real_sent_labels = [0] * len(src)
+ for l in real_sent_labels:
+ _real_sent_labels[l] = 1
+
+ # 增加一个real_labels变量 用于保存对应于src_str的label
+ real_labels = [_real_sent_labels[i] for i in idxs]
+
+ src = [src[i][:self.args.max_src_ntokens_per_sent] for i in idxs]
+ sent_labels = [_sent_labels[i] for i in idxs]
+ src = src[:self.args.max_src_nsents]
+ sent_labels = sent_labels[:self.args.max_src_nsents]
+
+ if (not is_test) and len(src) < self.args.min_src_nsents:
+ return None
+
+ src_txt = [' '.join(sent) for sent in src]
+ text = ' {} {} '.format(self.sep_token, self.cls_token).join(src_txt)
+
+ src_subtokens = self.tokenizer.tokenize(text)
+
+ src_subtokens = [self.cls_token] + src_subtokens + [self.sep_token]
+ src_subtoken_idxs = self.tokenizer.convert_tokens_to_ids(src_subtokens)
+ _segs = [-1] + [i for i, t in enumerate(src_subtoken_idxs) if t == self.sep_vid]
+ segs = [_segs[i] - _segs[i - 1] for i in range(1, len(_segs))]
+ segments_ids = []
+ for i, s in enumerate(segs):
+ if i % 2 == 0:
+ segments_ids += s * [0]
+ else:
+ segments_ids += s * [1]
+ cls_ids = [i for i, t in enumerate(src_subtoken_idxs) if t == self.cls_vid]
+ sent_labels = sent_labels[:len(cls_ids)]
+
+ tgt_subtokens_str = '[unused0] ' + ' [unused2] '.join(
+ [' '.join(self.tokenizer.tokenize(' '.join(tt), use_bert_basic_tokenizer=use_bert_basic_tokenizer)) for tt
+ in tgt]) + ' [unused1]'
+ tgt_subtoken = tgt_subtokens_str.split()[:self.args.max_tgt_ntokens]
+ if (not is_test) and len(tgt_subtoken) < self.args.min_tgt_ntokens:
+ return None
+
+ tgt_subtoken_idxs = self.tokenizer.convert_tokens_to_ids(tgt_subtoken)
+
+ tgt_txt = ''.join([' '.join(tt) for tt in tgt])
+ src_txt = [original_src_txt[i] for i in idxs]
+
+ return src_subtoken_idxs, sent_labels, tgt_subtoken_idxs, segments_ids, cls_ids, src_txt, tgt_txt, real_labels
+
+
+def format_to_bert_mp(args):
+ if args.dataset != '':
+ datasets = [args.dataset]
+ else:
+ datasets = ['train', 'val', 'test']
+
+ for corpus_type in datasets:
+ a_lst = []
+ for jsonl_f in glob.glob(pjoin(args.raw_path, '*' + corpus_type + '*.jsonl')):
+ real_name = jsonl_f.split('/')[-1]
+ a_lst.append((jsonl_f, pjoin(args.save_path, real_name.replace('jsonl', 'bert.jsonl'))))
+ if len(a_lst) != 1:
+ raise RuntimeError(f"文件夹里面包含多个命中为 *{corpus_type}*.jsonl 的文件或无此文件")
+
+ jsonl_file, save_file = a_lst[0]
+ logger.info('Processing %s' % jsonl_file)
+ jsonl_insts = load_jsonl(jsonl_file)
+ is_test = [(corpus_type == 'test')] * len(jsonl_insts)
+
+ with Pool(args.n_cpu) as p:
+ formatted_insts = p.map(format_to_bert, zip(jsonl_insts, is_test))
+ logger.info('Processed instances %d' % len(formatted_insts))
+ logger.info('Saving to %s' % save_file)
+
+ with open(save_file, "w") as f:
+ for inst in formatted_insts:
+ if inst is not None:
+ print(json.dumps(inst), file=f)
+ gc.collect()
+
+
+def format_to_bert(args):
+ logger.info("Process#: {}".format(os.getppid()))
+ inst, is_test = args
+
+ source = [sent.split() for sent in inst['text']]
+ tgt = [sent.split() for sent in inst['summary']]
+
+ sent_labels = greedy_selection(source[:parsed_args.max_src_nsents], tgt)
+ real_sent_labels = greedy_selection(source, tgt)
+ if parsed_args.lower:
+ source = [' '.join(s).lower().split() for s in source]
+ tgt = [' '.join(s).lower().split() for s in tgt]
+
+ b_data = bert.preprocess(source, tgt, sent_labels, real_sent_labels,
+ use_bert_basic_tokenizer=parsed_args.use_bert_basic_tokenizer,
+ is_test=is_test)
+
+ if b_data is None:
+ return None
+
+ src_subtoken_idxs, sent_labels, tgt_subtoken_idxs, segments_ids, cls_ids, src_txt, tgt_txt, real_labels = b_data
+ b_data_dict = {"text_ids": src_subtoken_idxs, "summary_ids": tgt_subtoken_idxs,
+ "label": sent_labels, "segment_ids": segments_ids, 'cls_ids': cls_ids,
+ 'text': src_txt, "summary": tgt_txt, 'real_label': real_labels}
+ return b_data_dict
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--raw_path', default='../jsonl_data')
+ parser.add_argument('--save_path', default='../data/')
+ # parser.add_argument('--has_label', default=True)
+ parser.add_argument('--min_src_nsents', default=3, type=int)
+ parser.add_argument('--max_src_nsents', default=50, type=int)
+ parser.add_argument('--min_src_ntokens_per_sent', default=5, type=int)
+ parser.add_argument('--max_src_ntokens_per_sent', default=50, type=int)
+ parser.add_argument('--min_tgt_ntokens', default=5, type=int)
+ parser.add_argument('--max_tgt_ntokens', default=400, type=int)
+
+ parser.add_argument("--lower", type=str2bool, nargs='?', const=True, default=False)
+ parser.add_argument("--use_bert_basic_tokenizer", type=str2bool, nargs='?', const=True, default=False)
+
+ parser.add_argument('--log_file', default='../logs/default.log')
+
+ parser.add_argument('--dataset', default='')
+
+ parser.add_argument('--n_cpu', default=32, type=int)
+
+ parsed_args = parser.parse_args()
+
+ logger = init_logger(parsed_args.log_file)
+ bert = BertData(parsed_args)
+
+ mkdir(parsed_args.save_path)
+ logger.info(time.clock())
+ format_to_bert_mp(parsed_args)
+ logger.info(time.clock())
diff --git a/fastSum/PreSum/train_abstractive.py b/fastSum/PreSum/train_abstractive.py
new file mode 100644
index 0000000000000000000000000000000000000000..8b1f6bc772dbb9152e9eb19232753486a067785d
--- /dev/null
+++ b/fastSum/PreSum/train_abstractive.py
@@ -0,0 +1,427 @@
+<<<<<<< HEAD
+#!/usr/bin/env python
+"""
+ Main training workflow
+"""
+from __future__ import division
+
+import argparse
+import glob
+import os
+import random
+import signal
+import time
+
+import torch
+from pytorch_transformers import BertTokenizer
+
+import distributed
+# from models import data_loader, model_builder
+# from models.data_loader import load_dataset
+# from models.loss import abs_loss
+from models.model_builder import AbsSummarizer
+# from models.predictor import build_predictor
+# from models.trainer import build_trainer
+from others.logging import logger, init_logger
+from others.utils import get_data_path, configure_training
+import torch.distributed as dist
+
+import json
+from os.path import join, exists
+from dataloader import PreSummABSLoader
+
+from models.optimizers import build_optim
+from metrics import MyNLLLoss, ABSLossMetric, FastRougeMetricABS, PyRougeMetricABS
+from callback import MyCallback, SaveModelCallback, EarlyStopCallback, FitlogCallback
+from fastNLP.core.trainer import Trainer
+from trainer import MyTrainer
+from fastNLP.core.tester import Tester
+from fastNLP import DistTrainer, get_local_rank
+
+model_flags = ['hidden_size', 'ff_size', 'heads', 'emb_size', 'enc_layers', 'enc_hidden_size', 'enc_ff_size',
+ 'dec_layers', 'dec_hidden_size', 'dec_ff_size', 'encoder', 'ff_actv', 'use_interval']
+
+
+def str2bool(v):
+ if v.lower() in ('yes', 'true', 't', 'y', '1'):
+ return True
+ elif v.lower() in ('no', 'false', 'f', 'n', '0'):
+ return False
+ else:
+ raise argparse.ArgumentTypeError('Boolean value expected.')
+
+
+#
+# def baseline(args, cal_lead=False, cal_oracle=False):
+# test_iter = data_loader.Dataloader(args, load_dataset(args, 'test', shuffle=False),
+# args.batch_size, 'cpu',
+# shuffle=False, is_test=True)
+#
+# trainer = build_trainer(args, '-1', None, None, None)
+# #
+# if (cal_lead):
+# trainer.test(test_iter, 0, cal_lead=True)
+# elif (cal_oracle):
+# trainer.test(test_iter, 0, cal_oracle=True)
+
+
+def train_abs(args):
+ init_logger(args.log_file)
+ logger.info(str(args))
+
+ # check if the data_path and save_path exists
+ data_paths = get_data_path(args.mode, args.label_type)
+ for name in data_paths:
+ assert exists(data_paths[name])
+ if not exists(args.save_path):
+ os.makedirs(args.save_path)
+
+ # device = "cpu" if args.visible_gpus == '-1' else "cuda"
+ # logger.info('Device ID %d' % device_id)
+ # logger.info('Device %s' % device)
+ # torch.manual_seed(args.seed)
+ # random.seed(args.seed)
+ # torch.backends.cudnn.deterministic = True
+ #
+ # if device_id >= 0:
+ # torch.cuda.set_device(device_id)
+ # torch.cuda.manual_seed(args.seed)
+ if args.local_rank not in [-1, 0]:
+ dist.barrier()
+
+ if args.train_from != '':
+ logger.info('Loading checkpoint from %s' % args.train_from)
+ checkpoint = torch.load(args.train_from,
+ map_location=lambda storage, loc: storage)
+ opt = vars(checkpoint['opt'])
+ for k in opt.keys():
+ if (k in model_flags):
+ setattr(args, k, opt[k])
+ else:
+ checkpoint = None
+
+ if (args.load_from_extractive != ''):
+ logger.info('Loading bert from extractive model %s' % args.load_from_extractive)
+ bert_from_extractive = torch.load(args.load_from_extractive, map_location=lambda storage, loc: storage)
+ bert_from_extractive = bert_from_extractive['model']
+ else:
+ bert_from_extractive = None
+
+ # torch.manual_seed(args.seed)
+ # random.seed(args.seed)
+ # torch.backends.cudnn.deterministic = True
+
+ # load summarization datasets
+ datasets = PreSummABSLoader(args).process(data_paths)
+ print('Information of dataset is:')
+ print(datasets)
+ train_set = datasets.datasets['train']
+ valid_set = datasets.datasets['val']
+
+ # configure training
+ devices, train_params = configure_training(args)
+ with open(join(args.save_path, 'params.json'), 'w') as f:
+ json.dump(train_params, f, indent=4)
+ print('Devices is:')
+ print(devices)
+
+ tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True, cache_dir=args.temp_dir)
+ symbols = {'BOS': tokenizer.vocab['[unused0]'], 'EOS': tokenizer.vocab['[unused1]'],
+ 'PAD': tokenizer.vocab['[PAD]'], 'EOQ': tokenizer.vocab['[unused2]']}
+
+ model = AbsSummarizer(args, checkpoint, bert_from_extractive, symbols, tokenizer)
+
+ logger.info(model)
+
+ # if (args.sep_optim):
+ # optim_bert = model_builder.build_optim_bert(args, model, checkpoint)
+ # optim_dec = model_builder.build_optim_dec(args, model, checkpoint)
+ # optim = [optim_bert, optim_dec]
+ # else:
+ # optim = [model_builder.build_optim(args, model, checkpoint)]
+
+ optim = build_optim(args, model, checkpoint)
+
+ if args.local_rank not in [-1, 0]:
+ dist.barrier()
+
+ # train_loss = abs_loss(model.generator, symbols, model.vocab_size, device, train=True,
+ # label_smoothing=args.label_smoothing)
+ train_loss = MyNLLLoss(model.generator, model.vocab_size, pred='pred', target='target',
+ label_smoothing=args.label_smoothing,
+ pad_id=symbols['PAD'])
+ callbacks = [MyCallback(args, optims=optim),
+ SaveModelCallback(args.save_path, optims=optim, args=args, save_on_exception=True),
+ EarlyStopCallback(), FitlogCallback(data=valid_set)]
+ val_metric = [ABSLossMetric(model.generator, model.vocab_size, pred='pred', target='target',
+ label_smoothing=args.label_smoothing,
+ pad_id=symbols['PAD'])]
+
+ trainer = MyTrainer(train_data=train_set, model=model, optimizer=None,
+ loss=train_loss, batch_size=args.batch_size, # sampler=sampler,
+ update_every=args.accum_count, n_epochs=args.n_epochs,
+ print_every=100, dev_data=valid_set, metrics=val_metric,
+ metric_key='-loss', validate_every=args.valid_steps * args.accum_count,
+ save_path=args.save_path, device=devices, callbacks=callbacks, config=args)
+
+ print('Start training with the following hyper-parameters:')
+ print(train_params)
+ trainer.train()
+
+ # trainer = build_trainer(args, device_id, model, optim, train_loss)
+ #
+ # trainer.train(train_iter_fct, args.train_steps)
+
+
+def test_abs(args, pt):
+ if (pt != ''):
+ test_from = pt
+ else:
+ test_from = args.test_from
+ logger.info('Loading checkpoint from %s' % test_from)
+
+ checkpoint = torch.load(test_from, map_location=lambda storage, loc: storage)
+ opt = vars(checkpoint['opt'])
+ for k in opt.keys():
+ if (k in model_flags):
+ setattr(args, k, opt[k])
+ print(args)
+
+ # load summarization datasets
+ data_paths = get_data_path(args.mode, args.label_type)
+ datasets = PreSummABSLoader(args).process(data_paths)
+ print('Information of dataset is:')
+ print(datasets)
+ test_set = datasets.datasets['test']
+
+ # only need 1 gpu for testing
+ device = int(args.visible_gpus)
+
+ tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True, cache_dir=args.temp_dir)
+ symbols = {'BOS': tokenizer.vocab['[unused0]'], 'EOS': tokenizer.vocab['[unused1]'],
+ 'PAD': tokenizer.vocab['[PAD]'], 'EOQ': tokenizer.vocab['[unused2]']}
+
+ model = AbsSummarizer(args, checkpoint=checkpoint, symbols=symbols, tokenizer=tokenizer)
+ model.eval()
+
+ test_metric = PyRougeMetricABS(pred='predictions', tgt_txt='tgt_txt', config=args, vocab=tokenizer, logger=logger)
+ tester = Tester(data=test_set, model=model, metrics=[test_metric],
+ batch_size=args.test_batch_size, device=device)
+ tester.test()
+
+ # predictor = build_predictor(args, tokenizer, symbols, model, logger)
+ # predictor.translate(test_iter, step)
+=======
+#!/usr/bin/env python
+"""
+ Main training workflow
+"""
+from __future__ import division
+
+import argparse
+import glob
+import os
+import random
+import signal
+import time
+
+import torch
+from pytorch_transformers import BertTokenizer
+
+import distributed
+# from models import data_loader, model_builder
+# from models.data_loader import load_dataset
+# from models.loss import abs_loss
+from models.model_builder import AbsSummarizer
+# from models.predictor import build_predictor
+# from models.trainer import build_trainer
+from others.logging import logger, init_logger
+from others.utils import get_data_path, configure_training
+import torch.distributed as dist
+
+import json
+from os.path import join, exists
+from dataloader import PreSummABSLoader
+
+from models.optimizers import build_optim
+from metrics import MyNLLLoss, ABSLossMetric, FastRougeMetricABS, PyRougeMetricABS
+from callback import MyCallback, SaveModelCallback, EarlyStopCallback, FitlogCallback
+from fastNLP.core.trainer import Trainer
+from trainer import MyTrainer
+from fastNLP.core.tester import Tester
+from fastNLP import DistTrainer, get_local_rank
+
+model_flags = ['hidden_size', 'ff_size', 'heads', 'emb_size', 'enc_layers', 'enc_hidden_size', 'enc_ff_size',
+ 'dec_layers', 'dec_hidden_size', 'dec_ff_size', 'encoder', 'ff_actv', 'use_interval']
+
+
+def str2bool(v):
+ if v.lower() in ('yes', 'true', 't', 'y', '1'):
+ return True
+ elif v.lower() in ('no', 'false', 'f', 'n', '0'):
+ return False
+ else:
+ raise argparse.ArgumentTypeError('Boolean value expected.')
+
+
+#
+# def baseline(args, cal_lead=False, cal_oracle=False):
+# test_iter = data_loader.Dataloader(args, load_dataset(args, 'test', shuffle=False),
+# args.batch_size, 'cpu',
+# shuffle=False, is_test=True)
+#
+# trainer = build_trainer(args, '-1', None, None, None)
+# #
+# if (cal_lead):
+# trainer.test(test_iter, 0, cal_lead=True)
+# elif (cal_oracle):
+# trainer.test(test_iter, 0, cal_oracle=True)
+
+
+def train_abs(args):
+ init_logger(args.log_file)
+ logger.info(str(args))
+
+ # check if the data_path and save_path exists
+ data_paths = get_data_path(args.mode, args.label_type)
+ for name in data_paths:
+ assert exists(data_paths[name])
+ if not exists(args.save_path):
+ os.makedirs(args.save_path)
+
+ # device = "cpu" if args.visible_gpus == '-1' else "cuda"
+ # logger.info('Device ID %d' % device_id)
+ # logger.info('Device %s' % device)
+ # torch.manual_seed(args.seed)
+ # random.seed(args.seed)
+ # torch.backends.cudnn.deterministic = True
+ #
+ # if device_id >= 0:
+ # torch.cuda.set_device(device_id)
+ # torch.cuda.manual_seed(args.seed)
+ if args.local_rank not in [-1, 0]:
+ dist.barrier()
+
+ if args.train_from != '':
+ logger.info('Loading checkpoint from %s' % args.train_from)
+ checkpoint = torch.load(args.train_from,
+ map_location=lambda storage, loc: storage)
+ opt = vars(checkpoint['opt'])
+ for k in opt.keys():
+ if (k in model_flags):
+ setattr(args, k, opt[k])
+ else:
+ checkpoint = None
+
+ if (args.load_from_extractive != ''):
+ logger.info('Loading bert from extractive model %s' % args.load_from_extractive)
+ bert_from_extractive = torch.load(args.load_from_extractive, map_location=lambda storage, loc: storage)
+ bert_from_extractive = bert_from_extractive['model']
+ else:
+ bert_from_extractive = None
+
+ # torch.manual_seed(args.seed)
+ # random.seed(args.seed)
+ # torch.backends.cudnn.deterministic = True
+
+ # load summarization datasets
+ datasets = PreSummABSLoader(args).process(data_paths)
+ print('Information of dataset is:')
+ print(datasets)
+ train_set = datasets.datasets['train']
+ valid_set = datasets.datasets['val']
+
+ # configure training
+ devices, train_params = configure_training(args)
+ with open(join(args.save_path, 'params.json'), 'w') as f:
+ json.dump(train_params, f, indent=4)
+ print('Devices is:')
+ print(devices)
+
+ tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True, cache_dir=args.temp_dir)
+ symbols = {'BOS': tokenizer.vocab['[unused0]'], 'EOS': tokenizer.vocab['[unused1]'],
+ 'PAD': tokenizer.vocab['[PAD]'], 'EOQ': tokenizer.vocab['[unused2]']}
+
+ model = AbsSummarizer(args, checkpoint, bert_from_extractive, symbols, tokenizer)
+
+ logger.info(model)
+
+ # if (args.sep_optim):
+ # optim_bert = model_builder.build_optim_bert(args, model, checkpoint)
+ # optim_dec = model_builder.build_optim_dec(args, model, checkpoint)
+ # optim = [optim_bert, optim_dec]
+ # else:
+ # optim = [model_builder.build_optim(args, model, checkpoint)]
+
+ optim = build_optim(args, model, checkpoint)
+
+ if args.local_rank not in [-1, 0]:
+ dist.barrier()
+
+ # train_loss = abs_loss(model.generator, symbols, model.vocab_size, device, train=True,
+ # label_smoothing=args.label_smoothing)
+ train_loss = MyNLLLoss(model.generator, model.vocab_size, pred='pred', target='target',
+ label_smoothing=args.label_smoothing,
+ pad_id=symbols['PAD'])
+ callbacks = [MyCallback(args, optims=optim),
+ SaveModelCallback(args.save_path, optims=optim, args=args, save_on_exception=True),
+ EarlyStopCallback(), FitlogCallback(data=valid_set)]
+ val_metric = [ABSLossMetric(model.generator, model.vocab_size, pred='pred', target='target',
+ label_smoothing=args.label_smoothing,
+ pad_id=symbols['PAD'])]
+
+ trainer = MyTrainer(train_data=train_set, model=model, optimizer=None,
+ loss=train_loss, batch_size=args.batch_size, # sampler=sampler,
+ update_every=args.accum_count, n_epochs=args.n_epochs,
+ print_every=100, dev_data=valid_set, metrics=val_metric,
+ metric_key='-loss', validate_every=args.valid_steps * args.accum_count,
+ save_path=args.save_path, device=devices, callbacks=callbacks, config=args)
+
+ print('Start training with the following hyper-parameters:')
+ print(train_params)
+ trainer.train()
+
+ # trainer = build_trainer(args, device_id, model, optim, train_loss)
+ #
+ # trainer.train(train_iter_fct, args.train_steps)
+
+
+def test_abs(args, pt):
+ if (pt != ''):
+ test_from = pt
+ else:
+ test_from = args.test_from
+ logger.info('Loading checkpoint from %s' % test_from)
+
+ checkpoint = torch.load(test_from, map_location=lambda storage, loc: storage)
+ opt = vars(checkpoint['opt'])
+ for k in opt.keys():
+ if (k in model_flags):
+ setattr(args, k, opt[k])
+ print(args)
+
+ # load summarization datasets
+ data_paths = get_data_path(args.mode, args.label_type)
+ datasets = PreSummABSLoader(args).process(data_paths)
+ print('Information of dataset is:')
+ print(datasets)
+ test_set = datasets.datasets['test']
+
+ # only need 1 gpu for testing
+ device = int(args.visible_gpus)
+
+ tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True, cache_dir=args.temp_dir)
+ symbols = {'BOS': tokenizer.vocab['[unused0]'], 'EOS': tokenizer.vocab['[unused1]'],
+ 'PAD': tokenizer.vocab['[PAD]'], 'EOQ': tokenizer.vocab['[unused2]']}
+
+ model = AbsSummarizer(args, checkpoint=checkpoint, symbols=symbols, tokenizer=tokenizer)
+ model.eval()
+
+ test_metric = PyRougeMetricABS(pred='predictions', tgt_txt='tgt_txt', config=args, vocab=tokenizer, logger=logger)
+ tester = Tester(data=test_set, model=model, metrics=[test_metric],
+ batch_size=args.test_batch_size, device=device)
+ tester.test()
+
+ # predictor = build_predictor(args, tokenizer, symbols, model, logger)
+ # predictor.translate(test_iter, step)
+>>>>>>> bcb618ae9faf5d35f3147b05b34268a73808238a
diff --git a/fastSum/PreSum/train_extractive.py b/fastSum/PreSum/train_extractive.py
new file mode 100644
index 0000000000000000000000000000000000000000..08f713a1fdcf29654615e37c5b7e212df5260c1e
--- /dev/null
+++ b/fastSum/PreSum/train_extractive.py
@@ -0,0 +1,321 @@
+<<<<<<< HEAD
+#!/usr/bin/env python
+"""
+ Main training workflow
+"""
+from __future__ import division
+
+import argparse
+import glob
+import os
+import random
+import signal
+import time
+
+import torch
+
+import distributed
+# from models import data_loader, model_builder
+# from models.data_loader import load_dataset
+from models.model_builder import ExtSummarizer
+# from models.trainer_ext import build_trainer
+from others.logging import logger, init_logger
+from others.utils import get_data_path, configure_training
+
+import json
+from os.path import join, exists
+
+from models.optimizers import build_optim
+from metrics import MyBCELoss, EXTLossMetric, PyRougeMetricEXT, FastRougeMetricEXT
+from callback import MyCallback, SaveModelCallback, EarlyStopCallback, FitlogCallback
+from fastNLP.core.trainer import Trainer
+from trainer import MyTrainer
+from fastNLP.core.tester import Tester
+from dataloader import PreSummEXTLoader
+
+model_flags = ['hidden_size', 'ff_size', 'heads', 'inter_layers', 'encoder', 'ff_actv', 'use_interval', 'rnn_size']
+
+
+#
+# def test_ext(args, device_id, pt, step):
+# device = "cpu" if args.visible_gpus == '-1' else "cuda"
+# if (pt != ''):
+# test_from = pt
+# else:
+# test_from = args.test_from
+# logger.info('Loading checkpoint from %s' % test_from)
+# checkpoint = torch.load(test_from, map_location=lambda storage, loc: storage)
+# opt = vars(checkpoint['opt'])
+# for k in opt.keys():
+# if (k in model_flags):
+# setattr(args, k, opt[k])
+# print(args)
+#
+# model = ExtSummarizer(args, device, checkpoint)
+# model.eval()
+#
+# test_iter = data_loader.Dataloader(args, load_dataset(args, 'test', shuffle=False),
+# args.test_batch_size, device,
+# shuffle=False, is_test=True)
+# trainer = build_trainer(args, device_id, model, None)
+# trainer.test(test_iter, step)
+
+
+def train_ext(args):
+ init_logger(args.log_file)
+ logger.info(str(args))
+
+ # check if the data_path and save_path exists
+ data_paths = get_data_path(args.mode, args.label_type)
+ for name in data_paths:
+ assert exists(data_paths[name])
+ if not exists(args.save_path):
+ os.makedirs(args.save_path)
+
+ if args.train_from != '':
+ logger.info('Loading checkpoint from %s' % args.train_from)
+ checkpoint = torch.load(args.train_from,
+ map_location=lambda storage, loc: storage)
+ opt = vars(checkpoint['opt'])
+ for k in opt.keys():
+ if (k in model_flags):
+ setattr(args, k, opt[k])
+ else:
+ checkpoint = None
+
+ # load summarization datasets
+ datasets = PreSummEXTLoader(args).process(data_paths)
+ print('Information of dataset is:')
+ print(datasets)
+ train_set = datasets.datasets['train']
+ valid_set = datasets.datasets['val']
+
+ # configure training
+ devices, train_params = configure_training(args)
+ with open(join(args.save_path, 'params.json'), 'w') as f:
+ json.dump(train_params, f, indent=4)
+ print('Devices is:')
+ print(devices)
+
+ model = ExtSummarizer(args, checkpoint)
+ optim = build_optim(args, model, checkpoint)
+
+ train_loss = MyBCELoss()
+ callbacks = [MyCallback(args, optims=optim),
+ SaveModelCallback(args.save_path, optims=optim, args=args, save_on_exception=True),
+ EarlyStopCallback(), FitlogCallback(data=valid_set)]
+ val_metric = [EXTLossMetric()]
+
+ logger.info(model)
+
+ trainer = MyTrainer(train_data=train_set, model=model, optimizer=None,
+ loss=train_loss, batch_size=args.batch_size, # sampler=sampler,
+ update_every=args.accum_count, n_epochs=args.n_epochs,
+ print_every=100, dev_data=valid_set, metrics=val_metric,
+ metric_key='-loss', validate_every=args.valid_steps * args.accum_count,
+ save_path=args.save_path, device=devices, callbacks=callbacks, config=args)
+
+ print('Start training with the following hyper-parameters:')
+ print(train_params)
+ trainer.train()
+
+ # trainer = build_trainer(args, device_id, model, optim)
+ # trainer.train(train_iter_fct, args.train_steps)
+
+
+def test_ext(args, pt):
+ if (pt != ''):
+ test_from = pt
+ else:
+ test_from = args.test_from
+ init_logger(args.log_file)
+ logger.info('Loading checkpoint from %s' % test_from)
+ checkpoint = torch.load(test_from, map_location=lambda storage, loc: storage)
+ opt = vars(checkpoint['opt'])
+ for k in opt.keys():
+ if (k in model_flags):
+ setattr(args, k, opt[k])
+ print(args)
+
+ # load summarization datasets
+ data_paths = get_data_path(args.mode, args.label_type)
+ datasets = PreSummEXTLoader(args).process(data_paths)
+ print('Information of dataset is:')
+ print(datasets)
+ test_set = datasets.datasets['test']
+
+ # only need 1 gpu for testing
+ device = int(args.visible_gpus)
+
+ model = ExtSummarizer(args, checkpoint)
+ model.eval()
+
+ test_metric = PyRougeMetricEXT(n_ext=3, ngram_block=3, pred='pred', src_txt='src_txt', tgt_txt='tgt_txt',
+ mask='mask', logger=logger, config=args)
+ tester = Tester(data=test_set, model=model, metrics=[test_metric],
+ batch_size=args.test_batch_size, device=device)
+ tester.test()
+
+ # trainer = build_trainer(args, device_id, model, None)
+ # trainer.test(test_iter, step)
+=======
+#!/usr/bin/env python
+"""
+ Main training workflow
+"""
+from __future__ import division
+
+import argparse
+import glob
+import os
+import random
+import signal
+import time
+
+import torch
+
+import distributed
+# from models import data_loader, model_builder
+# from models.data_loader import load_dataset
+from models.model_builder import ExtSummarizer
+# from models.trainer_ext import build_trainer
+from others.logging import logger, init_logger
+from others.utils import get_data_path, configure_training
+
+import json
+from os.path import join, exists
+
+from models.optimizers import build_optim
+from metrics import MyBCELoss, EXTLossMetric, PyRougeMetricEXT, FastRougeMetricEXT
+from callback import MyCallback, SaveModelCallback, EarlyStopCallback, FitlogCallback
+from fastNLP.core.trainer import Trainer
+from trainer import MyTrainer
+from fastNLP.core.tester import Tester
+from dataloader import PreSummEXTLoader
+
+model_flags = ['hidden_size', 'ff_size', 'heads', 'inter_layers', 'encoder', 'ff_actv', 'use_interval', 'rnn_size']
+
+
+#
+# def test_ext(args, device_id, pt, step):
+# device = "cpu" if args.visible_gpus == '-1' else "cuda"
+# if (pt != ''):
+# test_from = pt
+# else:
+# test_from = args.test_from
+# logger.info('Loading checkpoint from %s' % test_from)
+# checkpoint = torch.load(test_from, map_location=lambda storage, loc: storage)
+# opt = vars(checkpoint['opt'])
+# for k in opt.keys():
+# if (k in model_flags):
+# setattr(args, k, opt[k])
+# print(args)
+#
+# model = ExtSummarizer(args, device, checkpoint)
+# model.eval()
+#
+# test_iter = data_loader.Dataloader(args, load_dataset(args, 'test', shuffle=False),
+# args.test_batch_size, device,
+# shuffle=False, is_test=True)
+# trainer = build_trainer(args, device_id, model, None)
+# trainer.test(test_iter, step)
+
+
+def train_ext(args):
+ init_logger(args.log_file)
+ logger.info(str(args))
+
+ # check if the data_path and save_path exists
+ data_paths = get_data_path(args.mode, args.label_type)
+ for name in data_paths:
+ assert exists(data_paths[name])
+ if not exists(args.save_path):
+ os.makedirs(args.save_path)
+
+ if args.train_from != '':
+ logger.info('Loading checkpoint from %s' % args.train_from)
+ checkpoint = torch.load(args.train_from,
+ map_location=lambda storage, loc: storage)
+ opt = vars(checkpoint['opt'])
+ for k in opt.keys():
+ if (k in model_flags):
+ setattr(args, k, opt[k])
+ else:
+ checkpoint = None
+
+ # load summarization datasets
+ datasets = PreSummEXTLoader(args).process(data_paths)
+ print('Information of dataset is:')
+ print(datasets)
+ train_set = datasets.datasets['train']
+ valid_set = datasets.datasets['val']
+
+ # configure training
+ devices, train_params = configure_training(args)
+ with open(join(args.save_path, 'params.json'), 'w') as f:
+ json.dump(train_params, f, indent=4)
+ print('Devices is:')
+ print(devices)
+
+ model = ExtSummarizer(args, checkpoint)
+ optim = build_optim(args, model, checkpoint)
+
+ train_loss = MyBCELoss()
+ callbacks = [MyCallback(args, optims=optim),
+ SaveModelCallback(args.save_path, optims=optim, args=args, save_on_exception=True),
+ EarlyStopCallback(), FitlogCallback(data=valid_set)]
+ val_metric = [EXTLossMetric()]
+
+ logger.info(model)
+
+ trainer = MyTrainer(train_data=train_set, model=model, optimizer=None,
+ loss=train_loss, batch_size=args.batch_size, # sampler=sampler,
+ update_every=args.accum_count, n_epochs=args.n_epochs,
+ print_every=100, dev_data=valid_set, metrics=val_metric,
+ metric_key='-loss', validate_every=args.valid_steps * args.accum_count,
+ save_path=args.save_path, device=devices, callbacks=callbacks, config=args)
+
+ print('Start training with the following hyper-parameters:')
+ print(train_params)
+ trainer.train()
+
+ # trainer = build_trainer(args, device_id, model, optim)
+ # trainer.train(train_iter_fct, args.train_steps)
+
+
+def test_ext(args, pt):
+ if (pt != ''):
+ test_from = pt
+ else:
+ test_from = args.test_from
+ init_logger(args.log_file)
+ logger.info('Loading checkpoint from %s' % test_from)
+ checkpoint = torch.load(test_from, map_location=lambda storage, loc: storage)
+ opt = vars(checkpoint['opt'])
+ for k in opt.keys():
+ if (k in model_flags):
+ setattr(args, k, opt[k])
+ print(args)
+
+ # load summarization datasets
+ data_paths = get_data_path(args.mode, args.label_type)
+ datasets = PreSummEXTLoader(args).process(data_paths)
+ print('Information of dataset is:')
+ print(datasets)
+ test_set = datasets.datasets['test']
+
+ # only need 1 gpu for testing
+ device = int(args.visible_gpus)
+
+ model = ExtSummarizer(args, checkpoint)
+ model.eval()
+
+ test_metric = PyRougeMetricEXT(n_ext=3, ngram_block=3, pred='pred', src_txt='src_txt', tgt_txt='tgt_txt',
+ mask='mask', logger=logger, config=args)
+ tester = Tester(data=test_set, model=model, metrics=[test_metric],
+ batch_size=args.test_batch_size, device=device)
+ tester.test()
+
+ # trainer = build_trainer(args, device_id, model, None)
+ # trainer.test(test_iter, step)
+>>>>>>> bcb618ae9faf5d35f3147b05b34268a73808238a
diff --git a/fastSum/PreSum/train_presum.py b/fastSum/PreSum/train_presum.py
new file mode 100644
index 0000000000000000000000000000000000000000..5b033dc3af7a52d3b9ad1aa95075dd7d0221ed3e
--- /dev/null
+++ b/fastSum/PreSum/train_presum.py
@@ -0,0 +1,387 @@
+<<<<<<< HEAD
+#!/usr/bin/env python
+"""
+ Main training workflow
+"""
+from __future__ import division
+
+import argparse
+import os
+from others.logging import init_logger
+# from train_abstractive import validate_abs, train_abs, baseline, test_abs, test_text_abs
+# from train_extractive import train_ext, validate_ext, test_ext
+
+from train_abstractive import train_abs, test_abs
+from train_extractive import train_ext, test_ext
+import torch
+import torch.distributed as dist
+import numpy as np
+import random
+
+import fitlog
+
+fitlog.commit(__file__)
+fitlog.set_log_dir("fitlogs/")
+
+model_flags = ['hidden_size', 'ff_size', 'heads', 'emb_size', 'enc_layers', 'enc_hidden_size', 'enc_ff_size',
+ 'dec_layers', 'dec_hidden_size', 'dec_ff_size', 'encoder', 'ff_actv', 'use_interval']
+
+
+def str2bool(v):
+ if v.lower() in ('yes', 'true', 't', 'y', '1'):
+ return True
+ elif v.lower() in ('no', 'false', 'f', 'n', '0'):
+ return False
+ else:
+ raise argparse.ArgumentTypeError('Boolean value expected.')
+
+
+def set_random_seeds(random_seed):
+ torch.manual_seed(random_seed)
+ torch.backends.cudnn.deterministic = True
+ torch.backends.cudnn.benchmark = False
+ np.random.seed(random_seed)
+ random.seed(random_seed)
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser()
+ parser.add_argument("-task", default='ext', type=str, choices=['ext', 'abs'])
+ parser.add_argument("-encoder", default='bert', type=str, choices=['bert', 'baseline'])
+ parser.add_argument("-mode", default='train', type=str, choices=['train', 'val', 'test'])
+ # parser.add_argument("-bert_data_path", default='data/CNNDM_bert')
+ # parser.add_argument("-model_path", default='models')
+ # parser.add_argument("-result_path", default='results/cnndm')
+ parser.add_argument("-temp_dir", default='temp')
+
+ parser.add_argument("-batch_size", default=140, type=int)
+ parser.add_argument("-test_batch_size", default=200, type=int)
+
+ parser.add_argument("-max_pos", default=512, type=int)
+ # parser.add_argument("-use_interval", type=str2bool, nargs='?', const=True, default=True)
+ parser.add_argument("-large", type=str2bool, nargs='?', const=True, default=False)
+ parser.add_argument("-load_from_extractive", default='', type=str)
+
+ parser.add_argument("-sep_optim", type=str2bool, nargs='?', const=True, default=False)
+ parser.add_argument("-lr_bert", default=2e-3, type=float)
+ parser.add_argument("-lr_dec", default=2e-3, type=float)
+ parser.add_argument("-use_bert_emb", type=str2bool, nargs='?', const=True, default=False)
+
+ parser.add_argument("-share_emb", type=str2bool, nargs='?', const=True, default=False)
+ parser.add_argument("-finetune_bert", type=str2bool, nargs='?', const=True, default=True)
+ parser.add_argument("-dec_dropout", default=0.2, type=float)
+ parser.add_argument("-dec_layers", default=6, type=int)
+ parser.add_argument("-dec_hidden_size", default=768, type=int)
+ parser.add_argument("-dec_heads", default=8, type=int)
+ parser.add_argument("-dec_ff_size", default=2048, type=int)
+ parser.add_argument("-enc_hidden_size", default=768, type=int)
+ parser.add_argument("-enc_ff_size", default=512, type=int)
+ parser.add_argument("-enc_dropout", default=0.2, type=float)
+ parser.add_argument("-enc_layers", default=6, type=int)
+
+ # params for EXT
+ parser.add_argument("-ext_dropout", default=0.2, type=float)
+ parser.add_argument("-ext_layers", default=2, type=int)
+ parser.add_argument("-ext_hidden_size", default=768, type=int)
+ parser.add_argument("-ext_heads", default=8, type=int)
+ parser.add_argument("-ext_ff_size", default=2048, type=int)
+
+ parser.add_argument("-label_smoothing", default=0.1, type=float)
+ parser.add_argument("-generator_shard_size", default=32, type=int)
+ parser.add_argument("-alpha", default=0.6, type=float)
+ parser.add_argument("-beam_size", default=5, type=int)
+ parser.add_argument("-min_length", default=15, type=int)
+ parser.add_argument("-max_length", default=150, type=int)
+ parser.add_argument("-max_tgt_len", default=140, type=int)
+
+ parser.add_argument("-param_init", default=0, type=float)
+ parser.add_argument("-param_init_glorot", type=str2bool, nargs='?', const=True, default=True)
+ parser.add_argument("-optim", default='adam', type=str)
+ parser.add_argument("-lr", default=1, type=float)
+ parser.add_argument("-beta1", default=0.9, type=float)
+ parser.add_argument("-beta2", default=0.999, type=float)
+ parser.add_argument("-warmup_steps", default=8000, type=int)
+ parser.add_argument("-warmup_steps_bert", default=8000, type=int)
+ parser.add_argument("-warmup_steps_dec", default=8000, type=int)
+ parser.add_argument("-max_grad_norm", default=0, type=float)
+
+ parser.add_argument("-save_checkpoint_steps", default=5, type=int)
+ parser.add_argument("-accum_count", default=1, type=int)
+ # parser.add_argument("-report_every", default=1, type=int)
+ # parser.add_argument("-train_steps", default=1000, type=int)
+ parser.add_argument("-recall_eval", type=str2bool, nargs='?', const=True, default=False)
+
+ parser.add_argument('-visible_gpus', default='-1', type=str)
+ # parser.add_argument('-gpu_ranks', default='0', type=str)
+ parser.add_argument('-log_file', default='../logs/cnndm.log')
+ parser.add_argument('-seed', default=666, type=int)
+
+ # parser.add_argument("-test_all", type=str2bool, nargs='?', const=True, default=False)
+ parser.add_argument("-test_from", default='')
+ # parser.add_argument("-test_start_from", default=-1, type=int)
+
+ parser.add_argument("-train_from", default='')
+ # parser.add_argument("-report_rouge", type=str2bool, nargs='?', const=True, default=True)
+ parser.add_argument("-block_trigram", type=str2bool, nargs='?', const=True, default=True)
+ parser.add_argument("-label_type", type=str, required=True)
+ parser.add_argument("-save_path", type=str, default="checkpoints", help="path to saving the params")
+ parser.add_argument("-valid_steps", type=int, help="how much steps to perform validation")
+ parser.add_argument("-n_epochs", type=int)
+ parser.add_argument("-decode_path", type=str, default="results",
+ help="the path to save gold summary and generated summary, this is only needed when mode is test")
+ parser.add_argument("-max_summary_len", type=int, required=True, help="max summary length when loading dataset")
+ parser.add_argument('--local_rank', type=int, default=None)
+ parser.add_argument('--init_method', type=str, default='env://')
+
+ args = parser.parse_args()
+ # args.gpu_ranks = [int(i) for i in range(len(args.visible_gpus.split(',')))]
+ # args.world_size = len(args.gpu_ranks)
+ # os.environ["CUDA_VISIBLE_DEVICES"] = args.visible_gpus
+ set_random_seeds(args.seed)
+
+ # if args.local_rank is not None:
+ # torch.cuda.set_device(args.local_rank)
+ # dist.init_process_group("nccl", init_method=args.init_method)
+ # args.dist = True
+ # else:
+ # args.dist = False
+
+ init_logger(args.log_file)
+ # device = "cpu" if args.visible_gpus == '-1' else "cuda"
+ # device_id = 0 if device == "cuda" else -1
+ fitlog.add_hyper(args)
+ fitlog.add_hyper_in_file(__file__)
+
+ if (args.task == 'abs'):
+ if (args.mode == 'train'):
+ train_abs(args)
+ # elif (args.mode == 'val'):
+ # validate_abs(args, device_id)
+ # elif (args.mode == 'lead'):
+ # baseline(args, cal_lead=True)
+ # elif (args.mode == 'oracle'):
+ # baseline(args, cal_oracle=True)
+ if (args.mode == 'test'):
+ cp = args.test_from
+ if not os.path.exists(args.decode_path):
+ os.mkdir(args.decode_path)
+ test_abs(args, cp)
+ # elif (args.mode == 'test_text'):
+ # cp = args.test_from
+ # try:
+ # step = int(cp.split('.')[-2].split('_')[-1])
+ # except:
+ # step = 0
+ # test_text_abs(args, device_id, cp, step)
+
+ elif (args.task == 'ext'):
+ if (args.mode == 'train'):
+ train_ext(args)
+ # elif (args.mode == 'val'):
+ # validate_ext(args, device_id)
+ if (args.mode == 'test'):
+ cp = args.test_from
+ if not os.path.exists(args.decode_path):
+ os.mkdir(args.decode_path)
+ test_ext(args, cp)
+ # elif (args.mode == 'test_text'):
+ # cp = args.test_from
+ # try:
+ # step = int(cp.split('.')[-2].split('_')[-1])
+ # except:
+ # step = 0
+ # test_text_abs(args, device_id, cp, step)
+=======
+#!/usr/bin/env python
+"""
+ Main training workflow
+"""
+from __future__ import division
+
+import argparse
+import os
+from others.logging import init_logger
+# from train_abstractive import validate_abs, train_abs, baseline, test_abs, test_text_abs
+# from train_extractive import train_ext, validate_ext, test_ext
+
+from train_abstractive import train_abs, test_abs
+from train_extractive import train_ext, test_ext
+import torch
+import torch.distributed as dist
+import numpy as np
+import random
+
+import fitlog
+
+fitlog.commit(__file__)
+fitlog.set_log_dir("fitlogs/")
+
+model_flags = ['hidden_size', 'ff_size', 'heads', 'emb_size', 'enc_layers', 'enc_hidden_size', 'enc_ff_size',
+ 'dec_layers', 'dec_hidden_size', 'dec_ff_size', 'encoder', 'ff_actv', 'use_interval']
+
+
+def str2bool(v):
+ if v.lower() in ('yes', 'true', 't', 'y', '1'):
+ return True
+ elif v.lower() in ('no', 'false', 'f', 'n', '0'):
+ return False
+ else:
+ raise argparse.ArgumentTypeError('Boolean value expected.')
+
+
+def set_random_seeds(random_seed):
+ torch.manual_seed(random_seed)
+ torch.backends.cudnn.deterministic = True
+ torch.backends.cudnn.benchmark = False
+ np.random.seed(random_seed)
+ random.seed(random_seed)
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser()
+ parser.add_argument("-task", default='ext', type=str, choices=['ext', 'abs'])
+ parser.add_argument("-encoder", default='bert', type=str, choices=['bert', 'baseline'])
+ parser.add_argument("-mode", default='train', type=str, choices=['train', 'val', 'test'])
+ # parser.add_argument("-bert_data_path", default='data/CNNDM_bert')
+ # parser.add_argument("-model_path", default='models')
+ # parser.add_argument("-result_path", default='results/cnndm')
+ parser.add_argument("-temp_dir", default='temp')
+
+ parser.add_argument("-batch_size", default=140, type=int)
+ parser.add_argument("-test_batch_size", default=200, type=int)
+
+ parser.add_argument("-max_pos", default=512, type=int)
+ # parser.add_argument("-use_interval", type=str2bool, nargs='?', const=True, default=True)
+ parser.add_argument("-large", type=str2bool, nargs='?', const=True, default=False)
+ parser.add_argument("-load_from_extractive", default='', type=str)
+
+ parser.add_argument("-sep_optim", type=str2bool, nargs='?', const=True, default=False)
+ parser.add_argument("-lr_bert", default=2e-3, type=float)
+ parser.add_argument("-lr_dec", default=2e-3, type=float)
+ parser.add_argument("-use_bert_emb", type=str2bool, nargs='?', const=True, default=False)
+
+ parser.add_argument("-share_emb", type=str2bool, nargs='?', const=True, default=False)
+ parser.add_argument("-finetune_bert", type=str2bool, nargs='?', const=True, default=True)
+ parser.add_argument("-dec_dropout", default=0.2, type=float)
+ parser.add_argument("-dec_layers", default=6, type=int)
+ parser.add_argument("-dec_hidden_size", default=768, type=int)
+ parser.add_argument("-dec_heads", default=8, type=int)
+ parser.add_argument("-dec_ff_size", default=2048, type=int)
+ parser.add_argument("-enc_hidden_size", default=768, type=int)
+ parser.add_argument("-enc_ff_size", default=512, type=int)
+ parser.add_argument("-enc_dropout", default=0.2, type=float)
+ parser.add_argument("-enc_layers", default=6, type=int)
+
+ # params for EXT
+ parser.add_argument("-ext_dropout", default=0.2, type=float)
+ parser.add_argument("-ext_layers", default=2, type=int)
+ parser.add_argument("-ext_hidden_size", default=768, type=int)
+ parser.add_argument("-ext_heads", default=8, type=int)
+ parser.add_argument("-ext_ff_size", default=2048, type=int)
+
+ parser.add_argument("-label_smoothing", default=0.1, type=float)
+ parser.add_argument("-generator_shard_size", default=32, type=int)
+ parser.add_argument("-alpha", default=0.6, type=float)
+ parser.add_argument("-beam_size", default=5, type=int)
+ parser.add_argument("-min_length", default=15, type=int)
+ parser.add_argument("-max_length", default=150, type=int)
+ parser.add_argument("-max_tgt_len", default=140, type=int)
+
+ parser.add_argument("-param_init", default=0, type=float)
+ parser.add_argument("-param_init_glorot", type=str2bool, nargs='?', const=True, default=True)
+ parser.add_argument("-optim", default='adam', type=str)
+ parser.add_argument("-lr", default=1, type=float)
+ parser.add_argument("-beta1", default=0.9, type=float)
+ parser.add_argument("-beta2", default=0.999, type=float)
+ parser.add_argument("-warmup_steps", default=8000, type=int)
+ parser.add_argument("-warmup_steps_bert", default=8000, type=int)
+ parser.add_argument("-warmup_steps_dec", default=8000, type=int)
+ parser.add_argument("-max_grad_norm", default=0, type=float)
+
+ parser.add_argument("-save_checkpoint_steps", default=5, type=int)
+ parser.add_argument("-accum_count", default=1, type=int)
+ # parser.add_argument("-report_every", default=1, type=int)
+ # parser.add_argument("-train_steps", default=1000, type=int)
+ parser.add_argument("-recall_eval", type=str2bool, nargs='?', const=True, default=False)
+
+ parser.add_argument('-visible_gpus', default='-1', type=str)
+ # parser.add_argument('-gpu_ranks', default='0', type=str)
+ parser.add_argument('-log_file', default='../logs/cnndm.log')
+ parser.add_argument('-seed', default=666, type=int)
+
+ # parser.add_argument("-test_all", type=str2bool, nargs='?', const=True, default=False)
+ parser.add_argument("-test_from", default='')
+ # parser.add_argument("-test_start_from", default=-1, type=int)
+
+ parser.add_argument("-train_from", default='')
+ # parser.add_argument("-report_rouge", type=str2bool, nargs='?', const=True, default=True)
+ parser.add_argument("-block_trigram", type=str2bool, nargs='?', const=True, default=True)
+ parser.add_argument("-label_type", type=str, required=True)
+ parser.add_argument("-save_path", type=str, default="checkpoints", help="path to saving the params")
+ parser.add_argument("-valid_steps", type=int, help="how much steps to perform validation")
+ parser.add_argument("-n_epochs", type=int)
+ parser.add_argument("-decode_path", type=str, default="results",
+ help="the path to save gold summary and generated summary, this is only needed when mode is test")
+ parser.add_argument("-max_summary_len", type=int, required=True, help="max summary length when loading dataset")
+ parser.add_argument('--local_rank', type=int, default=None)
+ parser.add_argument('--init_method', type=str, default='env://')
+
+ args = parser.parse_args()
+ # args.gpu_ranks = [int(i) for i in range(len(args.visible_gpus.split(',')))]
+ # args.world_size = len(args.gpu_ranks)
+ # os.environ["CUDA_VISIBLE_DEVICES"] = args.visible_gpus
+ set_random_seeds(args.seed)
+
+ # if args.local_rank is not None:
+ # torch.cuda.set_device(args.local_rank)
+ # dist.init_process_group("nccl", init_method=args.init_method)
+ # args.dist = True
+ # else:
+ # args.dist = False
+
+ init_logger(args.log_file)
+ # device = "cpu" if args.visible_gpus == '-1' else "cuda"
+ # device_id = 0 if device == "cuda" else -1
+ fitlog.add_hyper(args)
+ fitlog.add_hyper_in_file(__file__)
+
+ if (args.task == 'abs'):
+ if (args.mode == 'train'):
+ train_abs(args)
+ # elif (args.mode == 'val'):
+ # validate_abs(args, device_id)
+ # elif (args.mode == 'lead'):
+ # baseline(args, cal_lead=True)
+ # elif (args.mode == 'oracle'):
+ # baseline(args, cal_oracle=True)
+ if (args.mode == 'test'):
+ cp = args.test_from
+ if not os.path.exists(args.decode_path):
+ os.mkdir(args.decode_path)
+ test_abs(args, cp)
+ # elif (args.mode == 'test_text'):
+ # cp = args.test_from
+ # try:
+ # step = int(cp.split('.')[-2].split('_')[-1])
+ # except:
+ # step = 0
+ # test_text_abs(args, device_id, cp, step)
+
+ elif (args.task == 'ext'):
+ if (args.mode == 'train'):
+ train_ext(args)
+ # elif (args.mode == 'val'):
+ # validate_ext(args, device_id)
+ if (args.mode == 'test'):
+ cp = args.test_from
+ if not os.path.exists(args.decode_path):
+ os.mkdir(args.decode_path)
+ test_ext(args, cp)
+ # elif (args.mode == 'test_text'):
+ # cp = args.test_from
+ # try:
+ # step = int(cp.split('.')[-2].split('_')[-1])
+ # except:
+ # step = 0
+ # test_text_abs(args, device_id, cp, step)
+>>>>>>> bcb618ae9faf5d35f3147b05b34268a73808238a
diff --git a/fastSum/PreSum/trainer.py b/fastSum/PreSum/trainer.py
new file mode 100644
index 0000000000000000000000000000000000000000..3e5d5ffbb65c5aae197b4cd934cdc5bf5e175dde
--- /dev/null
+++ b/fastSum/PreSum/trainer.py
@@ -0,0 +1,859 @@
+<<<<<<< HEAD
+from fastNLP.core.trainer import Trainer
+import torch
+from torch import nn
+import os
+import numpy as np
+import warnings
+import time
+from datetime import datetime, timedelta
+
+try:
+ from tqdm.auto import tqdm
+except:
+ from fastNLP.core.utils import _pseudo_tqdm as tqdm
+
+from fastNLP.core.batch import DataSetIter, BatchIter
+from fastNLP.core.callback import CallbackManager, CallbackException, Callback
+from fastNLP.core.dataset import DataSet
+from fastNLP.core.losses import _prepare_losser
+from fastNLP.core.metrics import _prepare_metrics
+from fastNLP.core.optimizer import Optimizer
+from fastNLP.core.sampler import Sampler
+from fastNLP.core.sampler import RandomSampler
+from fastNLP.core.tester import Tester
+from fastNLP.core.utils import _CheckError
+from fastNLP.core.utils import _build_args
+from fastNLP.core.utils import _check_forward_error
+from fastNLP.core.utils import _check_loss_evaluate
+from fastNLP.core.utils import _move_dict_value_to_device
+from fastNLP.core.utils import _get_func_signature
+from fastNLP.core.utils import _get_model_device
+from fastNLP.core.utils import _move_model_to_device
+from fastNLP.core._parallel_utils import _model_contains_inner_module
+from fastNLP.core._logger import logger
+
+DEFAULT_CHECK_BATCH_SIZE = 2
+DEFAULT_CHECK_NUM_BATCH = 2
+
+
+def _model_contains_inner_module(model):
+ r"""
+
+ :param nn.Module model: 模型文件，判断是否内部包含model.module, 多用于check模型是否是nn.DataParallel,
+ nn.parallel.DistributedDataParallel。主要是在做形参匹配的时候需要使用最内部的model的function。
+ :return: bool
+ """
+ if isinstance(model, nn.Module):
+ if isinstance(model, (nn.DataParallel, nn.parallel.DistributedDataParallel)):
+ return True
+ return False
+
+
+class MyTrainer(Trainer):
+ def __init__(self, train_data, model, optimizer=None, loss=None,
+ batch_size=32, sampler=None, drop_last=False, update_every=1,
+ num_workers=0, n_epochs=10, print_every=5,
+ dev_data=None, metrics=None, metric_key=None,
+ validate_every=-1, save_path=None, use_tqdm=True, device=None,
+ callbacks=None, check_code_level=0, **kwargs):
+ r"""
+ :param train_data: 训练集， :class:`~fastNLP.DataSet` 类型或 :class:`~fastNLP.BatchIter` 的子类
+ :param nn.modules model: 待训练的模型
+ :param optimizer: `torch.optim.Optimizer` 优化器。如果为None，则Trainer使用默认的Adam(model.parameters(), lr=4e-3)这个优化器
+ :param int batch_size: 训练和验证的时候的batch大小。
+ :param loss: 使用的 :class:`~fastNLP.core.losses.LossBase` 对象。当为None时，默认使用 :class:`~fastNLP.LossInForward`
+ :param sampler: Batch数据生成的顺序， :class:`~fastNLP.Sampler` 类型。如果为None，默认使用 :class:`~fastNLP.RandomSampler`
+ :param drop_last: 如果最后一个batch没有正好为batch_size这么多数据，就扔掉最后一个batch
+ :param num_workers: int, 有多少个线程来进行数据pad处理。
+ :param update_every: int, 多少步更新一次梯度。用于希望累计梯度的场景，比如需要128的batch_size, 但是直接设为128
+ 会导致内存不足，通过设置batch_size=32, update_every=4达到目的。当optimizer为None时，该参数无效。
+ :param int n_epochs: 需要优化迭代多少次。
+ :param int print_every: 多少次反向传播更新tqdm显示的loss; 如果use_tqdm=False, 则多少次反向传播打印loss。
+ :param dev_data: 用于做验证的DataSet， :class:`~fastNLP.DataSet` 类型。
+ :param metrics: 验证的评估函数。可以只使用一个 :class:`Metric` ，
+ 也可以使用多个 :class:`Metric` ，通过列表传入。
+ 如验证时取得了更好的验证结果(如果有多个Metric，以列表中第一个Metric为准)，且save_path不为None，
+ 则保存当前模型。Metric种类详见 :mod:`metrics模块 ` 。仅在传入dev_data时有效。
+ :param str,None metric_key: :class:`Metric` 有时会有多个指标，
+ 比如 :class:`~fastNLP.core.metrics.SpanFPreRecMetric` 中包含了'f', 'pre', 'rec'。此时需
+ 要指定以哪个指标为准。另外有些指标是越小效果越好，比如语言模型的困惑度，这种情况下，在key前面增加一个'-'来表
+ 明验证时，值越小越好(比如: "-ppl")。仅在传入dev_data时有效。
+ :param int validate_every: 多少个step在验证集上验证一次; 如果为-1，则每个epoch结束验证一次。仅在传入dev_data时有效。
+ :param str,None save_path: 将模型保存路径，如果路径不存在，将自动创建文件夹。如果为None，则不保存模型。如果dev_data为None，则保存
+ 最后一次迭代的模型。保存的时候不仅保存了参数，还保存了模型结构。即便使用DataParallel，这里也只保存模型。
+ :param bool use_tqdm: 是否使用tqdm来显示训练进度; 如果为False，则将loss打印在终端中。
+ :param str,int,torch.device,list(int) device: 将模型load到哪个设备。默认为None，即Trainer不对模型
+ 的计算位置进行管理。支持以下的输入:
+
+ 1. str: ['cpu', 'cuda', 'cuda:0', 'cuda:1', ...] 依次为'cpu'中, 可见的第一个GPU中, 可见的第一个GPU中,
+ 可见的第二个GPU中;
+
+ 2. torch.device：将模型装载到torch.device上。
+
+ 3. int: 将使用device_id为该值的gpu进行训练
+
+ 4. list(int)：如果多于1个device，将使用torch.nn.DataParallel包裹model, 并使用传入的device。
+
+ 5. None. 为None则不对模型进行任何处理，如果传入的model为torch.nn.DataParallel该值必须为None。
+
+ 已知可能会出现的问题：Adagrad优化器可能无法正常使用这个参数，请手动管理模型位置。
+
+ :param list(callbacks) callbacks: 用于在train过程中起调节作用的回调函数。比如early stop，negative sampling等可以
+ 通过callback机制实现。 可使用的callback参见 :mod:`callback模块 `
+ :param int check_code_level: 模型检查等级. -1: 不进行检查; 0: 仅出现错误时停止; 1: 如果有field没有被使用，
+ 报告警告信息; 2: 有任何field没有被使用都报错. 检查的原理是通过使用很小的batch(默认2个sample)来运行代码，但是
+ 这个过程理论上不会修改任何参数，只是会检查能否运行。但如果(1)模型中存在将batch_size写为某个固定值的情况；
+ (2)模型中存在累加前向计算次数的，可能会多计算1次。以上情况建议将check_code_level设置为-1。
+ :param kwargs: 支持配置可选参数
+ bool test_use_tqdm: 在dev上验证的时候是否开启tqdm
+ Sampler test_sampler: 在evaluate的时候使用的sampler
+
+ 本trainer是对FastNLP原始Trainer的修改，修改原因：
+ 1. 原始trainer的保存模型格式和my-presum保存模型格式不符，导致best performance的checkpoint和在固定step保存的checkpoint格式不一致
+ 2. 原始trainer默认如果传入的optimizer是None，会自动分配一个Adam的optimizer，并不符合presum中需要有两个optimizer的情况
+ 修改内容：
+ 1. 修改_save_model和_load_model函数，使得和本项目内部存储格式一致
+ 2. 修改Trainer的初始化以及_update()函数，使得如果optimizer是None，则trainer不进行self.optimizer.step(), 对应工作由自己设定的callback进行
+ """
+ super(Trainer, self).__init__()
+ if not isinstance(model, nn.Module):
+ raise TypeError(f"The type of model must be torch.nn.Module, got {type(model)}.")
+
+ # check metrics and dev_data
+ if (not metrics) and dev_data is not None:
+ raise ValueError("No metric for dev_data evaluation.")
+ if metrics and (dev_data is None):
+ raise ValueError("No dev_data for evaluations, pass dev_data or set metrics to None. ")
+
+ # check update every
+ assert update_every >= 1, "update_every must be no less than 1."
+ self.update_every = int(update_every)
+
+ # check save_path
+ if not (save_path is None or isinstance(save_path, str)):
+ raise ValueError("save_path can only be None or `str`.")
+ # prepare evaluate
+ metrics = _prepare_metrics(metrics)
+
+ # parse metric_key
+ # increase_better is True. It means the exp result gets better if the indicator increases.
+ # It is true by default.
+ self.increase_better = True
+ if metric_key is not None:
+ self.increase_better = False if metric_key[0] == "-" else True
+ self.metric_key = metric_key[1:] if metric_key[0] == "+" or metric_key[0] == "-" else metric_key
+ else:
+ self.metric_key = None
+ # prepare loss
+ losser = _prepare_losser(loss)
+
+ if isinstance(train_data, BatchIter):
+ if sampler is not None:
+ warnings.warn("sampler is ignored when train_data is a BatchIter.")
+ if num_workers > 0:
+ warnings.warn("num_workers is ignored when train_data is BatchIter.")
+ if drop_last:
+ warnings.warn("drop_last is ignored when train_data is BatchIter.")
+
+ if isinstance(model, nn.parallel.DistributedDataParallel): # 如果是分布式的
+ # device为None
+ if device is not None:
+ warnings.warn("device is ignored when model is nn.parallel.DistributedDataParallel.")
+ device = None
+ # Sampler要是分布式的
+ if sampler is None:
+ sampler = torch.utils.data.DistributedSampler(train_data)
+ elif not isinstance(sampler, torch.utils.data.DistributedSampler):
+ raise TypeError("When using nn.parallel.DistributedDataParallel, "
+ "sampler must be None or torch.utils.data.DistributedSampler.")
+ # 不能保存模型
+ if save_path:
+ raise RuntimeError("Saving model in Distributed situation is not allowed right now.")
+ else:
+ # sampler check
+ if sampler is not None and not isinstance(sampler, (Sampler, torch.utils.data.Sampler)):
+ raise ValueError(
+ f"The type of sampler should be fastNLP.BaseSampler or pytorch's Sampler, got {type(sampler)}")
+ if sampler is None:
+ sampler = RandomSampler()
+ elif hasattr(sampler, 'set_batch_size'):
+ sampler.set_batch_size(batch_size)
+
+ if isinstance(train_data, DataSet):
+ self.data_iterator = DataSetIter(dataset=train_data, batch_size=batch_size, sampler=sampler,
+ num_workers=num_workers, drop_last=drop_last)
+ elif isinstance(train_data, BatchIter):
+ self.data_iterator = train_data
+ train_data = train_data.dataset
+ check_code_level = -1 # 强制跳过校验
+ else:
+ raise TypeError("train_data type {} not support".format(type(train_data)))
+
+ model.train()
+ self.model = _move_model_to_device(model, device=device)
+ if _model_contains_inner_module(self.model):
+ self._forward_func = self.model.module.forward
+ else:
+ self._forward_func = self.model.forward
+ if check_code_level > -1:
+ # _check_code 是 fastNLP 帮助你检查代码是否正确的方法 。如果你在错误栈中看到这行注释，请认真检查你的field名与模型的输入
+ # 名是否匹配
+ dev_dataset = dev_data
+ if isinstance(dev_data, BatchIter):
+ dev_dataset = None
+ warnings.warn("dev_data is of BatchIter type, ignore validation checking.")
+ check_batch_size = min(batch_size, DEFAULT_CHECK_BATCH_SIZE)
+ if isinstance(self.model, nn.DataParallel):
+ _num_devices = len(self.model.device_ids)
+ if batch_size // _num_devices > 1: # 如果多卡是每个卡可以分多个数据的，则用每个卡给两个sample
+ check_batch_size = max(len(self.model.device_ids) * 2, check_batch_size)
+ else:
+ check_batch_size = max(len(self.model.device_ids), check_batch_size)
+ _check_code(dataset=train_data, model=self.model, losser=losser, forward_func=self._forward_func,
+ metrics=metrics,
+ dev_data=dev_dataset, metric_key=self.metric_key, check_level=check_code_level,
+ batch_size=check_batch_size)
+
+ self.train_data = train_data
+ self.dev_data = dev_data # If None, No validation.
+ self.losser = losser
+ self.metrics = metrics
+ self.n_epochs = int(n_epochs)
+ self.batch_size = int(batch_size)
+ self.save_path = save_path
+ self.print_every = int(print_every)
+ self.validate_every = int(validate_every) if validate_every != 0 else -1
+ self.best_metric_indicator = None
+ self.best_dev_epoch = None
+ self.best_dev_step = None
+ self.best_dev_perf = None
+ self.n_steps = len(self.data_iterator) * self.n_epochs
+
+ if isinstance(optimizer, torch.optim.Optimizer):
+ self.optimizer = optimizer
+ elif isinstance(optimizer, Optimizer):
+ self.optimizer = optimizer.construct_from_pytorch(self.model.parameters())
+ elif optimizer is None:
+ # 修改optimizer初始化，使得其不再是如果传入None则默认Adam
+ # self.optimizer = torch.optim.Adam(self.model.parameters(), lr=4e-3)
+ self.optimizer = None
+ else:
+ raise TypeError("optimizer can only be torch.optim.Optimizer type, not {}.".format(type(optimizer)))
+
+ self.logger = logger
+
+ self.use_tqdm = use_tqdm
+ if 'test_use_tqdm' in kwargs:
+ self.test_use_tqdm = kwargs.get('test_use_tqdm')
+ else:
+ self.test_use_tqdm = self.use_tqdm
+ self.pbar = None
+ self.print_every = abs(self.print_every)
+ self.kwargs = kwargs
+ if self.dev_data is not None:
+ self.tester = Tester(model=self.model,
+ data=self.dev_data,
+ metrics=self.metrics,
+ batch_size=kwargs.get("dev_batch_size", self.batch_size),
+ device=None, # 由上面的部分处理device
+ verbose=0,
+ use_tqdm=self.test_use_tqdm,
+ sampler=kwargs.get('test_sampler', None))
+
+ self.start_time = None # start timestamp
+
+ if isinstance(callbacks, Callback):
+ callbacks = [callbacks]
+
+ self.callback_manager = CallbackManager(env={"trainer": self},
+ callbacks=callbacks)
+
+ def _update(self):
+ r"""Perform weight update on a model.
+ 修改：增加了只有当self.optimizer is not None的时候才会进行self.optimizer.step()
+ """
+ if self.step % self.update_every == 0 and self.optimizer is not None:
+ self.optimizer.step()
+
+ def _save_model(self, model, model_name, only_param=True):
+ r""" 存储不含有显卡信息的state_dict或model
+ :param model:
+ :param model_name:
+ :param only_param:
+ :return:
+ """
+ config = self.kwargs.get("config")
+ assert config is not None, "Please pass config to MyTrainer!"
+
+ if self.save_path is not None:
+ model_path = os.path.join(self.save_path, model_name)
+ if not os.path.exists(self.save_path):
+ os.makedirs(self.save_path, exist_ok=True)
+ if _model_contains_inner_module(model):
+ model = model.module
+
+ if only_param:
+ state_dict = model.state_dict()
+ for key in state_dict:
+ state_dict[key] = state_dict[key].cpu()
+ checkpoint = {'model': state_dict, 'opt': config}
+ torch.save(checkpoint, model_path)
+ else:
+ model.cpu()
+ checkpoint = {'model': model, 'opt': config}
+ torch.save(checkpoint, model_path)
+ model.to(self._model_device)
+
+ def _load_model(self, model, model_name, only_param=True):
+ # 返回bool值指示是否成功reload模型
+ if self.save_path is not None:
+ model_path = os.path.join(self.save_path, model_name)
+ if only_param:
+ states = torch.load(model_path)['model']
+ else:
+ states = torch.load(model_path)['model'].state_dict()
+ if _model_contains_inner_module(model):
+ model.module.load_state_dict(states)
+ else:
+ model.load_state_dict(states)
+ elif hasattr(self, "_best_model_states"):
+ model.load_state_dict(self._best_model_states)
+ else:
+ return False
+ return True
+
+
+def _get_value_info(_dict):
+ # given a dict value, return information about this dict's value. Return list of str
+ strs = []
+ for key, value in _dict.items():
+ _str = ''
+ if isinstance(value, torch.Tensor):
+ _str += "\t{}: (1)type:torch.Tensor (2)dtype:{}, (3)shape:{} ".format(key,
+ value.dtype, value.size())
+ elif isinstance(value, np.ndarray):
+ _str += "\t{}: (1)type:numpy.ndarray (2)dtype:{}, (3)shape:{} ".format(key,
+ value.dtype, value.shape)
+ else:
+ _str += "\t{}: type:{}".format(key, type(value))
+ strs.append(_str)
+ return strs
+
+
+def _check_code(dataset, model, losser, metrics, forward_func, batch_size=DEFAULT_CHECK_BATCH_SIZE,
+ dev_data=None, metric_key=None, check_level=0):
+ # check get_loss 方法
+ model_device = _get_model_device(model=model)
+ _iter = DataSetIter(dataset, batch_size=batch_size, sampler=None)
+
+ for batch_count, (batch_x, batch_y) in enumerate(_iter):
+ _move_dict_value_to_device(batch_x, batch_y, device=model_device)
+ # forward check
+ if batch_count == 0:
+ info_str = ""
+ input_fields = _get_value_info(batch_x)
+ target_fields = _get_value_info(batch_y)
+ if len(input_fields) > 0:
+ info_str += "input fields after batch(if batch size is {}):\n".format(batch_size)
+ info_str += "\n".join(input_fields)
+ info_str += '\n'
+ else:
+ raise RuntimeError("There is no input field.")
+ if len(target_fields) > 0:
+ info_str += "target fields after batch(if batch size is {}):\n".format(batch_size)
+ info_str += "\n".join(target_fields)
+ info_str += '\n'
+ else:
+ info_str += 'There is no target field.'
+ logger.info(info_str)
+ _check_forward_error(forward_func=forward_func, dataset=dataset,
+ batch_x=batch_x, check_level=check_level)
+ refined_batch_x = _build_args(forward_func, **batch_x)
+ pred_dict = model(**refined_batch_x)
+ func_signature = _get_func_signature(forward_func)
+ if not isinstance(pred_dict, dict):
+ raise TypeError(f"The return value of {func_signature} should be `dict`, not `{type(pred_dict)}`.")
+
+ # loss check
+ try:
+ loss = losser(pred_dict, batch_y)
+ # check loss output
+ if batch_count == 0:
+ if not isinstance(loss, torch.Tensor):
+ raise TypeError(
+ f"The return value of {_get_func_signature(losser.get_loss)} should be `torch.Tensor`, "
+ f"but got `{type(loss)}`.")
+ if len(loss.size()) != 0:
+ raise ValueError(
+ f"The size of return value of {_get_func_signature(losser.get_loss)} is {loss.size()}, "
+ f"should be torch.size([])")
+ loss.backward()
+ except _CheckError as e:
+ # TODO: another error raised if _CheckError caught
+ pre_func_signature = _get_func_signature(forward_func)
+ _check_loss_evaluate(prev_func_signature=pre_func_signature, func_signature=e.func_signature,
+ check_res=e.check_res, pred_dict=pred_dict, target_dict=batch_y,
+ dataset=dataset, check_level=check_level)
+ model.zero_grad()
+ if batch_count + 1 >= DEFAULT_CHECK_NUM_BATCH:
+ break
+
+ if dev_data is not None:
+ tester = Tester(data=dev_data[:batch_size * DEFAULT_CHECK_NUM_BATCH], model=model, metrics=metrics,
+ batch_size=batch_size, verbose=-1, use_tqdm=False)
+ evaluate_results = tester.test()
+ _check_eval_results(metrics=evaluate_results, metric_key=metric_key, metric_list=metrics)
+
+
+def _check_eval_results(metrics, metric_key, metric_list):
+ # metrics: tester返回的结果
+ # metric_key: 一个用来做筛选的指标，来自Trainer的初始化
+ # metric_list: 多个用来做评价的指标，来自Trainer的初始化
+ if isinstance(metrics, tuple):
+ loss, metrics = metrics
+
+ if isinstance(metrics, dict):
+ metric_dict = list(metrics.values())[0] # 取第一个metric
+
+ if metric_key is None:
+ indicator_val, indicator = list(metric_dict.values())[0], list(metric_dict.keys())[0]
+ else:
+ # metric_key is set
+ if metric_key not in metric_dict:
+ raise RuntimeError(f"metric key {metric_key} not found in {metric_dict}")
+ indicator_val = metric_dict[metric_key]
+ indicator = metric_key
+ else:
+ raise RuntimeError("Invalid metrics type. Expect {}, got {}".format((tuple, dict), type(metrics)))
+ return indicator, indicator_val
+=======
+from fastNLP.core.trainer import Trainer
+import torch
+from torch import nn
+import os
+import numpy as np
+import warnings
+import time
+from datetime import datetime, timedelta
+
+try:
+ from tqdm.auto import tqdm
+except:
+ from fastNLP.core.utils import _pseudo_tqdm as tqdm
+
+from fastNLP.core.batch import DataSetIter, BatchIter
+from fastNLP.core.callback import CallbackManager, CallbackException, Callback
+from fastNLP.core.dataset import DataSet
+from fastNLP.core.losses import _prepare_losser
+from fastNLP.core.metrics import _prepare_metrics
+from fastNLP.core.optimizer import Optimizer
+from fastNLP.core.sampler import Sampler
+from fastNLP.core.sampler import RandomSampler
+from fastNLP.core.tester import Tester
+from fastNLP.core.utils import _CheckError
+from fastNLP.core.utils import _build_args
+from fastNLP.core.utils import _check_forward_error
+from fastNLP.core.utils import _check_loss_evaluate
+from fastNLP.core.utils import _move_dict_value_to_device
+from fastNLP.core.utils import _get_func_signature
+from fastNLP.core.utils import _get_model_device
+from fastNLP.core.utils import _move_model_to_device
+from fastNLP.core._parallel_utils import _model_contains_inner_module
+from fastNLP.core._logger import logger
+
+DEFAULT_CHECK_BATCH_SIZE = 2
+DEFAULT_CHECK_NUM_BATCH = 2
+
+
+def _model_contains_inner_module(model):
+ r"""
+
+ :param nn.Module model: 模型文件，判断是否内部包含model.module, 多用于check模型是否是nn.DataParallel,
+ nn.parallel.DistributedDataParallel。主要是在做形参匹配的时候需要使用最内部的model的function。
+ :return: bool
+ """
+ if isinstance(model, nn.Module):
+ if isinstance(model, (nn.DataParallel, nn.parallel.DistributedDataParallel)):
+ return True
+ return False
+
+
+class MyTrainer(Trainer):
+ def __init__(self, train_data, model, optimizer=None, loss=None,
+ batch_size=32, sampler=None, drop_last=False, update_every=1,
+ num_workers=0, n_epochs=10, print_every=5,
+ dev_data=None, metrics=None, metric_key=None,
+ validate_every=-1, save_path=None, use_tqdm=True, device=None,
+ callbacks=None, check_code_level=0, **kwargs):
+ r"""
+ :param train_data: 训练集， :class:`~fastNLP.DataSet` 类型或 :class:`~fastNLP.BatchIter` 的子类
+ :param nn.modules model: 待训练的模型
+ :param optimizer: `torch.optim.Optimizer` 优化器。如果为None，则Trainer使用默认的Adam(model.parameters(), lr=4e-3)这个优化器
+ :param int batch_size: 训练和验证的时候的batch大小。
+ :param loss: 使用的 :class:`~fastNLP.core.losses.LossBase` 对象。当为None时，默认使用 :class:`~fastNLP.LossInForward`
+ :param sampler: Batch数据生成的顺序， :class:`~fastNLP.Sampler` 类型。如果为None，默认使用 :class:`~fastNLP.RandomSampler`
+ :param drop_last: 如果最后一个batch没有正好为batch_size这么多数据，就扔掉最后一个batch
+ :param num_workers: int, 有多少个线程来进行数据pad处理。
+ :param update_every: int, 多少步更新一次梯度。用于希望累计梯度的场景，比如需要128的batch_size, 但是直接设为128
+ 会导致内存不足，通过设置batch_size=32, update_every=4达到目的。当optimizer为None时，该参数无效。
+ :param int n_epochs: 需要优化迭代多少次。
+ :param int print_every: 多少次反向传播更新tqdm显示的loss; 如果use_tqdm=False, 则多少次反向传播打印loss。
+ :param dev_data: 用于做验证的DataSet， :class:`~fastNLP.DataSet` 类型。
+ :param metrics: 验证的评估函数。可以只使用一个 :class:`Metric` ，
+ 也可以使用多个 :class:`Metric` ，通过列表传入。
+ 如验证时取得了更好的验证结果(如果有多个Metric，以列表中第一个Metric为准)，且save_path不为None，
+ 则保存当前模型。Metric种类详见 :mod:`metrics模块 ` 。仅在传入dev_data时有效。
+ :param str,None metric_key: :class:`Metric` 有时会有多个指标，
+ 比如 :class:`~fastNLP.core.metrics.SpanFPreRecMetric` 中包含了'f', 'pre', 'rec'。此时需
+ 要指定以哪个指标为准。另外有些指标是越小效果越好，比如语言模型的困惑度，这种情况下，在key前面增加一个'-'来表
+ 明验证时，值越小越好(比如: "-ppl")。仅在传入dev_data时有效。
+ :param int validate_every: 多少个step在验证集上验证一次; 如果为-1，则每个epoch结束验证一次。仅在传入dev_data时有效。
+ :param str,None save_path: 将模型保存路径，如果路径不存在，将自动创建文件夹。如果为None，则不保存模型。如果dev_data为None，则保存
+ 最后一次迭代的模型。保存的时候不仅保存了参数，还保存了模型结构。即便使用DataParallel，这里也只保存模型。
+ :param bool use_tqdm: 是否使用tqdm来显示训练进度; 如果为False，则将loss打印在终端中。
+ :param str,int,torch.device,list(int) device: 将模型load到哪个设备。默认为None，即Trainer不对模型
+ 的计算位置进行管理。支持以下的输入:
+
+ 1. str: ['cpu', 'cuda', 'cuda:0', 'cuda:1', ...] 依次为'cpu'中, 可见的第一个GPU中, 可见的第一个GPU中,
+ 可见的第二个GPU中;
+
+ 2. torch.device：将模型装载到torch.device上。
+
+ 3. int: 将使用device_id为该值的gpu进行训练
+
+ 4. list(int)：如果多于1个device，将使用torch.nn.DataParallel包裹model, 并使用传入的device。
+
+ 5. None. 为None则不对模型进行任何处理，如果传入的model为torch.nn.DataParallel该值必须为None。
+
+ 已知可能会出现的问题：Adagrad优化器可能无法正常使用这个参数，请手动管理模型位置。
+
+ :param list(callbacks) callbacks: 用于在train过程中起调节作用的回调函数。比如early stop，negative sampling等可以
+ 通过callback机制实现。 可使用的callback参见 :mod:`callback模块 `
+ :param int check_code_level: 模型检查等级. -1: 不进行检查; 0: 仅出现错误时停止; 1: 如果有field没有被使用，
+ 报告警告信息; 2: 有任何field没有被使用都报错. 检查的原理是通过使用很小的batch(默认2个sample)来运行代码，但是
+ 这个过程理论上不会修改任何参数，只是会检查能否运行。但如果(1)模型中存在将batch_size写为某个固定值的情况；
+ (2)模型中存在累加前向计算次数的，可能会多计算1次。以上情况建议将check_code_level设置为-1。
+ :param kwargs: 支持配置可选参数
+ bool test_use_tqdm: 在dev上验证的时候是否开启tqdm
+ Sampler test_sampler: 在evaluate的时候使用的sampler
+
+ 本trainer是对FastNLP原始Trainer的修改，修改原因：
+ 1. 原始trainer的保存模型格式和my-presum保存模型格式不符，导致best performance的checkpoint和在固定step保存的checkpoint格式不一致
+ 2. 原始trainer默认如果传入的optimizer是None，会自动分配一个Adam的optimizer，并不符合presum中需要有两个optimizer的情况
+ 修改内容：
+ 1. 修改_save_model和_load_model函数，使得和本项目内部存储格式一致
+ 2. 修改Trainer的初始化以及_update()函数，使得如果optimizer是None，则trainer不进行self.optimizer.step(), 对应工作由自己设定的callback进行
+ """
+ super(Trainer, self).__init__()
+ if not isinstance(model, nn.Module):
+ raise TypeError(f"The type of model must be torch.nn.Module, got {type(model)}.")
+
+ # check metrics and dev_data
+ if (not metrics) and dev_data is not None:
+ raise ValueError("No metric for dev_data evaluation.")
+ if metrics and (dev_data is None):
+ raise ValueError("No dev_data for evaluations, pass dev_data or set metrics to None. ")
+
+ # check update every
+ assert update_every >= 1, "update_every must be no less than 1."
+ self.update_every = int(update_every)
+
+ # check save_path
+ if not (save_path is None or isinstance(save_path, str)):
+ raise ValueError("save_path can only be None or `str`.")
+ # prepare evaluate
+ metrics = _prepare_metrics(metrics)
+
+ # parse metric_key
+ # increase_better is True. It means the exp result gets better if the indicator increases.
+ # It is true by default.
+ self.increase_better = True
+ if metric_key is not None:
+ self.increase_better = False if metric_key[0] == "-" else True
+ self.metric_key = metric_key[1:] if metric_key[0] == "+" or metric_key[0] == "-" else metric_key
+ else:
+ self.metric_key = None
+ # prepare loss
+ losser = _prepare_losser(loss)
+
+ if isinstance(train_data, BatchIter):
+ if sampler is not None:
+ warnings.warn("sampler is ignored when train_data is a BatchIter.")
+ if num_workers > 0:
+ warnings.warn("num_workers is ignored when train_data is BatchIter.")
+ if drop_last:
+ warnings.warn("drop_last is ignored when train_data is BatchIter.")
+
+ if isinstance(model, nn.parallel.DistributedDataParallel): # 如果是分布式的
+ # device为None
+ if device is not None:
+ warnings.warn("device is ignored when model is nn.parallel.DistributedDataParallel.")
+ device = None
+ # Sampler要是分布式的
+ if sampler is None:
+ sampler = torch.utils.data.DistributedSampler(train_data)
+ elif not isinstance(sampler, torch.utils.data.DistributedSampler):
+ raise TypeError("When using nn.parallel.DistributedDataParallel, "
+ "sampler must be None or torch.utils.data.DistributedSampler.")
+ # 不能保存模型
+ if save_path:
+ raise RuntimeError("Saving model in Distributed situation is not allowed right now.")
+ else:
+ # sampler check
+ if sampler is not None and not isinstance(sampler, (Sampler, torch.utils.data.Sampler)):
+ raise ValueError(
+ f"The type of sampler should be fastNLP.BaseSampler or pytorch's Sampler, got {type(sampler)}")
+ if sampler is None:
+ sampler = RandomSampler()
+ elif hasattr(sampler, 'set_batch_size'):
+ sampler.set_batch_size(batch_size)
+
+ if isinstance(train_data, DataSet):
+ self.data_iterator = DataSetIter(dataset=train_data, batch_size=batch_size, sampler=sampler,
+ num_workers=num_workers, drop_last=drop_last)
+ elif isinstance(train_data, BatchIter):
+ self.data_iterator = train_data
+ train_data = train_data.dataset
+ check_code_level = -1 # 强制跳过校验
+ else:
+ raise TypeError("train_data type {} not support".format(type(train_data)))
+
+ model.train()
+ self.model = _move_model_to_device(model, device=device)
+ if _model_contains_inner_module(self.model):
+ self._forward_func = self.model.module.forward
+ else:
+ self._forward_func = self.model.forward
+ if check_code_level > -1:
+ # _check_code 是 fastNLP 帮助你检查代码是否正确的方法 。如果你在错误栈中看到这行注释，请认真检查你的field名与模型的输入
+ # 名是否匹配
+ dev_dataset = dev_data
+ if isinstance(dev_data, BatchIter):
+ dev_dataset = None
+ warnings.warn("dev_data is of BatchIter type, ignore validation checking.")
+ check_batch_size = min(batch_size, DEFAULT_CHECK_BATCH_SIZE)
+ if isinstance(self.model, nn.DataParallel):
+ _num_devices = len(self.model.device_ids)
+ if batch_size // _num_devices > 1: # 如果多卡是每个卡可以分多个数据的，则用每个卡给两个sample
+ check_batch_size = max(len(self.model.device_ids) * 2, check_batch_size)
+ else:
+ check_batch_size = max(len(self.model.device_ids), check_batch_size)
+ _check_code(dataset=train_data, model=self.model, losser=losser, forward_func=self._forward_func,
+ metrics=metrics,
+ dev_data=dev_dataset, metric_key=self.metric_key, check_level=check_code_level,
+ batch_size=check_batch_size)
+
+ self.train_data = train_data
+ self.dev_data = dev_data # If None, No validation.
+ self.losser = losser
+ self.metrics = metrics
+ self.n_epochs = int(n_epochs)
+ self.batch_size = int(batch_size)
+ self.save_path = save_path
+ self.print_every = int(print_every)
+ self.validate_every = int(validate_every) if validate_every != 0 else -1
+ self.best_metric_indicator = None
+ self.best_dev_epoch = None
+ self.best_dev_step = None
+ self.best_dev_perf = None
+ self.n_steps = len(self.data_iterator) * self.n_epochs
+
+ if isinstance(optimizer, torch.optim.Optimizer):
+ self.optimizer = optimizer
+ elif isinstance(optimizer, Optimizer):
+ self.optimizer = optimizer.construct_from_pytorch(self.model.parameters())
+ elif optimizer is None:
+ # 修改optimizer初始化，使得其不再是如果传入None则默认Adam
+ # self.optimizer = torch.optim.Adam(self.model.parameters(), lr=4e-3)
+ self.optimizer = None
+ else:
+ raise TypeError("optimizer can only be torch.optim.Optimizer type, not {}.".format(type(optimizer)))
+
+ self.logger = logger
+
+ self.use_tqdm = use_tqdm
+ if 'test_use_tqdm' in kwargs:
+ self.test_use_tqdm = kwargs.get('test_use_tqdm')
+ else:
+ self.test_use_tqdm = self.use_tqdm
+ self.pbar = None
+ self.print_every = abs(self.print_every)
+ self.kwargs = kwargs
+ if self.dev_data is not None:
+ self.tester = Tester(model=self.model,
+ data=self.dev_data,
+ metrics=self.metrics,
+ batch_size=kwargs.get("dev_batch_size", self.batch_size),
+ device=None, # 由上面的部分处理device
+ verbose=0,
+ use_tqdm=self.test_use_tqdm,
+ sampler=kwargs.get('test_sampler', None))
+
+ self.start_time = None # start timestamp
+
+ if isinstance(callbacks, Callback):
+ callbacks = [callbacks]
+
+ self.callback_manager = CallbackManager(env={"trainer": self},
+ callbacks=callbacks)
+
+ def _update(self):
+ r"""Perform weight update on a model.
+ 修改：增加了只有当self.optimizer is not None的时候才会进行self.optimizer.step()
+ """
+ if self.step % self.update_every == 0 and self.optimizer is not None:
+ self.optimizer.step()
+
+ def _save_model(self, model, model_name, only_param=True):
+ r""" 存储不含有显卡信息的state_dict或model
+ :param model:
+ :param model_name:
+ :param only_param:
+ :return:
+ """
+ config = self.kwargs.get("config")
+ assert config is not None, "Please pass config to MyTrainer!"
+
+ if self.save_path is not None:
+ model_path = os.path.join(self.save_path, model_name)
+ if not os.path.exists(self.save_path):
+ os.makedirs(self.save_path, exist_ok=True)
+ if _model_contains_inner_module(model):
+ model = model.module
+
+ if only_param:
+ state_dict = model.state_dict()
+ for key in state_dict:
+ state_dict[key] = state_dict[key].cpu()
+ checkpoint = {'model': state_dict, 'opt': config}
+ torch.save(checkpoint, model_path)
+ else:
+ model.cpu()
+ checkpoint = {'model': model, 'opt': config}
+ torch.save(checkpoint, model_path)
+ model.to(self._model_device)
+
+ def _load_model(self, model, model_name, only_param=True):
+ # 返回bool值指示是否成功reload模型
+ if self.save_path is not None:
+ model_path = os.path.join(self.save_path, model_name)
+ if only_param:
+ states = torch.load(model_path)['model']
+ else:
+ states = torch.load(model_path)['model'].state_dict()
+ if _model_contains_inner_module(model):
+ model.module.load_state_dict(states)
+ else:
+ model.load_state_dict(states)
+ elif hasattr(self, "_best_model_states"):
+ model.load_state_dict(self._best_model_states)
+ else:
+ return False
+ return True
+
+
+def _get_value_info(_dict):
+ # given a dict value, return information about this dict's value. Return list of str
+ strs = []
+ for key, value in _dict.items():
+ _str = ''
+ if isinstance(value, torch.Tensor):
+ _str += "\t{}: (1)type:torch.Tensor (2)dtype:{}, (3)shape:{} ".format(key,
+ value.dtype, value.size())
+ elif isinstance(value, np.ndarray):
+ _str += "\t{}: (1)type:numpy.ndarray (2)dtype:{}, (3)shape:{} ".format(key,
+ value.dtype, value.shape)
+ else:
+ _str += "\t{}: type:{}".format(key, type(value))
+ strs.append(_str)
+ return strs
+
+
+def _check_code(dataset, model, losser, metrics, forward_func, batch_size=DEFAULT_CHECK_BATCH_SIZE,
+ dev_data=None, metric_key=None, check_level=0):
+ # check get_loss 方法
+ model_device = _get_model_device(model=model)
+ _iter = DataSetIter(dataset, batch_size=batch_size, sampler=None)
+
+ for batch_count, (batch_x, batch_y) in enumerate(_iter):
+ _move_dict_value_to_device(batch_x, batch_y, device=model_device)
+ # forward check
+ if batch_count == 0:
+ info_str = ""
+ input_fields = _get_value_info(batch_x)
+ target_fields = _get_value_info(batch_y)
+ if len(input_fields) > 0:
+ info_str += "input fields after batch(if batch size is {}):\n".format(batch_size)
+ info_str += "\n".join(input_fields)
+ info_str += '\n'
+ else:
+ raise RuntimeError("There is no input field.")
+ if len(target_fields) > 0:
+ info_str += "target fields after batch(if batch size is {}):\n".format(batch_size)
+ info_str += "\n".join(target_fields)
+ info_str += '\n'
+ else:
+ info_str += 'There is no target field.'
+ logger.info(info_str)
+ _check_forward_error(forward_func=forward_func, dataset=dataset,
+ batch_x=batch_x, check_level=check_level)
+ refined_batch_x = _build_args(forward_func, **batch_x)
+ pred_dict = model(**refined_batch_x)
+ func_signature = _get_func_signature(forward_func)
+ if not isinstance(pred_dict, dict):
+ raise TypeError(f"The return value of {func_signature} should be `dict`, not `{type(pred_dict)}`.")
+
+ # loss check
+ try:
+ loss = losser(pred_dict, batch_y)
+ # check loss output
+ if batch_count == 0:
+ if not isinstance(loss, torch.Tensor):
+ raise TypeError(
+ f"The return value of {_get_func_signature(losser.get_loss)} should be `torch.Tensor`, "
+ f"but got `{type(loss)}`.")
+ if len(loss.size()) != 0:
+ raise ValueError(
+ f"The size of return value of {_get_func_signature(losser.get_loss)} is {loss.size()}, "
+ f"should be torch.size([])")
+ loss.backward()
+ except _CheckError as e:
+ # TODO: another error raised if _CheckError caught
+ pre_func_signature = _get_func_signature(forward_func)
+ _check_loss_evaluate(prev_func_signature=pre_func_signature, func_signature=e.func_signature,
+ check_res=e.check_res, pred_dict=pred_dict, target_dict=batch_y,
+ dataset=dataset, check_level=check_level)
+ model.zero_grad()
+ if batch_count + 1 >= DEFAULT_CHECK_NUM_BATCH:
+ break
+
+ if dev_data is not None:
+ tester = Tester(data=dev_data[:batch_size * DEFAULT_CHECK_NUM_BATCH], model=model, metrics=metrics,
+ batch_size=batch_size, verbose=-1, use_tqdm=False)
+ evaluate_results = tester.test()
+ _check_eval_results(metrics=evaluate_results, metric_key=metric_key, metric_list=metrics)
+
+
+def _check_eval_results(metrics, metric_key, metric_list):
+ # metrics: tester返回的结果
+ # metric_key: 一个用来做筛选的指标，来自Trainer的初始化
+ # metric_list: 多个用来做评价的指标，来自Trainer的初始化
+ if isinstance(metrics, tuple):
+ loss, metrics = metrics
+
+ if isinstance(metrics, dict):
+ metric_dict = list(metrics.values())[0] # 取第一个metric
+
+ if metric_key is None:
+ indicator_val, indicator = list(metric_dict.values())[0], list(metric_dict.keys())[0]
+ else:
+ # metric_key is set
+ if metric_key not in metric_dict:
+ raise RuntimeError(f"metric key {metric_key} not found in {metric_dict}")
+ indicator_val = metric_dict[metric_key]
+ indicator = metric_key
+ else:
+ raise RuntimeError("Invalid metrics type. Expect {}, got {}".format((tuple, dict), type(metrics)))
+ return indicator, indicator_val
+>>>>>>> bcb618ae9faf5d35f3147b05b34268a73808238a
diff --git a/fastSum/PreSum/translate/__init__.py b/fastSum/PreSum/translate/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/fastSum/PreSum/translate/beam.py b/fastSum/PreSum/translate/beam.py
new file mode 100644
index 0000000000000000000000000000000000000000..daedc60d18e3812ef12ddd33deed52fc879fe29f
--- /dev/null
+++ b/fastSum/PreSum/translate/beam.py
@@ -0,0 +1,208 @@
+from __future__ import division
+import torch
+from translate import penalties
+
+
+class Beam(object):
+ """
+ Class for managing the internals of the beam search process.
+
+ Takes care of beams, back pointers, and scores.
+
+ Args:
+ size (int): beam size
+ pad, bos, eos (int): indices of padding, beginning, and ending.
+ n_best (int): nbest size to use
+ cuda (bool): use gpu
+ global_scorer (:obj:`GlobalScorer`)
+ """
+
+ def __init__(self, size, pad, bos, eos,
+ n_best=1, cuda=False,
+ global_scorer=None,
+ min_length=0,
+ stepwise_penalty=False,
+ block_ngram_repeat=0,
+ exclusion_tokens=set()):
+
+ self.size = size
+ self.tt = torch.cuda if cuda else torch
+
+ # The score for each translation on the beam.
+ self.scores = self.tt.FloatTensor(size).zero_()
+ self.all_scores = []
+
+ # The backpointers at each time-step.
+ self.prev_ks = []
+
+ # The outputs at each time-step.
+ self.next_ys = [self.tt.LongTensor(size)
+ .fill_(pad)]
+ self.next_ys[0][0] = bos
+
+ # Has EOS topped the beam yet.
+ self._eos = eos
+ self.eos_top = False
+
+ # The attentions (matrix) for each time.
+ self.attn = []
+
+ # Time and k pair for finished.
+ self.finished = []
+ self.n_best = n_best
+
+ # Information for global scoring.
+ self.global_scorer = global_scorer
+ self.global_state = {}
+
+ # Minimum prediction length
+ self.min_length = min_length
+
+ # Apply Penalty at every step
+ self.stepwise_penalty = stepwise_penalty
+ self.block_ngram_repeat = block_ngram_repeat
+ self.exclusion_tokens = exclusion_tokens
+
+ def get_current_state(self):
+ "Get the outputs for the current timestep."
+ return self.next_ys[-1]
+
+ def get_current_origin(self):
+ "Get the backpointers for the current timestep."
+ return self.prev_ks[-1]
+
+ def advance(self, word_probs, attn_out):
+ """
+ Given prob over words for every last beam `wordLk` and attention
+ `attn_out`: Compute and update the beam search.
+
+ Parameters:
+
+ * `word_probs`- probs of advancing from the last step (K x words)
+ * `attn_out`- attention at the last step
+
+ Returns: True if beam search is complete.
+ """
+ num_words = word_probs.size(1)
+ if self.stepwise_penalty:
+ self.global_scorer.update_score(self, attn_out)
+ # force the output to be longer than self.min_length
+ cur_len = len(self.next_ys)
+ if cur_len < self.min_length:
+ for k in range(len(word_probs)):
+ word_probs[k][self._eos] = -1e20
+ # Sum the previous scores.
+ if len(self.prev_ks) > 0:
+ beam_scores = word_probs + \
+ self.scores.unsqueeze(1).expand_as(word_probs)
+ # Don't let EOS have children.
+ for i in range(self.next_ys[-1].size(0)):
+ if self.next_ys[-1][i] == self._eos:
+ beam_scores[i] = -1e20
+
+ # Block ngram repeats
+ if self.block_ngram_repeat > 0:
+ ngrams = []
+ le = len(self.next_ys)
+ for j in range(self.next_ys[-1].size(0)):
+ hyp, _ = self.get_hyp(le - 1, j)
+ ngrams = set()
+ fail = False
+ gram = []
+ for i in range(le - 1):
+ # Last n tokens, n = block_ngram_repeat
+ gram = (gram +
+ [hyp[i].item()])[-self.block_ngram_repeat:]
+ # Skip the blocking if it is in the exclusion list
+ if set(gram) & self.exclusion_tokens:
+ continue
+ if tuple(gram) in ngrams:
+ fail = True
+ ngrams.add(tuple(gram))
+ if fail:
+ beam_scores[j] = -10e20
+ else:
+ beam_scores = word_probs[0]
+ flat_beam_scores = beam_scores.view(-1)
+ best_scores, best_scores_id = flat_beam_scores.topk(self.size, 0,
+ True, True)
+
+ self.all_scores.append(self.scores)
+ self.scores = best_scores
+
+ # best_scores_id is flattened beam x word array, so calculate which
+ # word and beam each score came from
+ prev_k = best_scores_id / num_words
+ self.prev_ks.append(prev_k)
+ self.next_ys.append((best_scores_id - prev_k * num_words))
+ self.attn.append(attn_out.index_select(0, prev_k))
+ self.global_scorer.update_global_state(self)
+
+ for i in range(self.next_ys[-1].size(0)):
+ if self.next_ys[-1][i] == self._eos:
+ global_scores = self.global_scorer.score(self, self.scores)
+ s = global_scores[i]
+ self.finished.append((s, len(self.next_ys) - 1, i))
+
+ # End condition is when top-of-beam is EOS and no global score.
+ if self.next_ys[-1][0] == self._eos:
+ self.all_scores.append(self.scores)
+ self.eos_top = True
+
+ def done(self):
+ return self.eos_top and len(self.finished) >= self.n_best
+
+ def sort_finished(self, minimum=None):
+ if minimum is not None:
+ i = 0
+ # Add from beam until we have minimum outputs.
+ while len(self.finished) < minimum:
+ global_scores = self.global_scorer.score(self, self.scores)
+ s = global_scores[i]
+ self.finished.append((s, len(self.next_ys) - 1, i))
+ i += 1
+
+ self.finished.sort(key=lambda a: -a[0])
+ scores = [sc for sc, _, _ in self.finished]
+ ks = [(t, k) for _, t, k in self.finished]
+ return scores, ks
+
+ def get_hyp(self, timestep, k):
+ """
+ Walk back to construct the full hypothesis.
+ """
+ hyp, attn = [], []
+ for j in range(len(self.prev_ks[:timestep]) - 1, -1, -1):
+ hyp.append(self.next_ys[j + 1][k])
+ attn.append(self.attn[j][k])
+ k = self.prev_ks[j][k]
+ return hyp[::-1], torch.stack(attn[::-1])
+
+
+class GNMTGlobalScorer(object):
+ """
+ NMT re-ranking score from
+ "Google's Neural Machine Translation System" :cite:`wu2016google`
+
+ Args:
+ alpha (float): length parameter
+ beta (float): coverage parameter
+ """
+
+ def __init__(self, alpha, length_penalty):
+ self.alpha = alpha
+ penalty_builder = penalties.PenaltyBuilder(length_penalty)
+ # Term will be subtracted from probability
+ # Probability will be divided by this
+ self.length_penalty = penalty_builder.length_penalty()
+
+ def score(self, beam, logprobs):
+ """
+ Rescores a prediction based on penalty functions
+ """
+ normalized_probs = self.length_penalty(beam,
+ logprobs,
+ self.alpha)
+
+ return normalized_probs
+
diff --git a/fastSum/PreSum/translate/penalties.py b/fastSum/PreSum/translate/penalties.py
new file mode 100644
index 0000000000000000000000000000000000000000..e2afbb291f777e60ed20148532d3d6e8906c53de
--- /dev/null
+++ b/fastSum/PreSum/translate/penalties.py
@@ -0,0 +1,50 @@
+from __future__ import division
+import torch
+
+
+class PenaltyBuilder(object):
+ """
+ Returns the Length and Coverage Penalty function for Beam Search.
+
+ Args:
+ length_pen (str): option name of length pen
+ cov_pen (str): option name of cov pen
+ """
+
+ def __init__(self, length_pen):
+ self.length_pen = length_pen
+
+ def length_penalty(self):
+ if self.length_pen == "wu":
+ return self.length_wu
+ elif self.length_pen == "avg":
+ return self.length_average
+ else:
+ return self.length_none
+
+ """
+ Below are all the different penalty terms implemented so far
+ """
+
+
+ def length_wu(self, beam, logprobs, alpha=0.):
+ """
+ NMT length re-ranking score from
+ "Google's Neural Machine Translation System" :cite:`wu2016google`.
+ """
+
+ modifier = (((5 + len(beam.next_ys)) ** alpha) /
+ ((5 + 1) ** alpha))
+ return (logprobs / modifier)
+
+ def length_average(self, beam, logprobs, alpha=0.):
+ """
+ Returns the average probability of tokens in a sequence.
+ """
+ return logprobs / len(beam.next_ys)
+
+ def length_none(self, beam, logprobs, alpha=0., beta=0.):
+ """
+ Returns unmodified scores.
+ """
+ return logprobs
\ No newline at end of file
diff --git a/fastSum/resources/ROUGE/DB_File-1.835.tar.gz b/fastSum/resources/ROUGE/DB_File-1.835.tar.gz
new file mode 100644
index 0000000000000000000000000000000000000000..36177e3a15cfd1c17f01d3ec981cccd6521a2cd0
Binary files /dev/null and b/fastSum/resources/ROUGE/DB_File-1.835.tar.gz differ
diff --git a/fastSum/resources/ROUGE/DB_File-1.835/Changes b/fastSum/resources/ROUGE/DB_File-1.835/Changes
new file mode 100644
index 0000000000000000000000000000000000000000..e653816fa2144974d9b4d17468f9289641c86e2b
--- /dev/null
+++ b/fastSum/resources/ROUGE/DB_File-1.835/Changes
@@ -0,0 +1,644 @@
+
+1.835 23 Dec 2014
+
+ * Silence more compiler warnings
+
+1.834 11 Dec 2014
+
+ * Makefile.PL: version check is missing a zero
+ RT #100844
+
+1.833 9 Dec 2014
+
+ * More Silence compiler warnings
+
+ * 1.832 breaks bleadperl C89 build
+ RT #100812
+
+1.832 8 Dec 2014
+
+ * Silence compiler warnings
+
+ * C++ change from blead
+
+1.831 15 November 2013
+
+ * C99 comment is a nogo
+ RT #90383
+
+1.830 2 November 2013
+
+ * Memory leaks when failed to open db
+ RT #89589
+
+ * DB_File uses AutoLoader for no reason
+ RT #88258
+
+1.829 7 July 2013
+
+ * make realclean: removing all files
+ RT #68214
+
+ * Documented the issue where the error below
+
+ BDB0588 At least one secondary cursor must be specified to DB->join
+
+ * DB_File installs to wrong place for CPAN version
+ RT #70420
+ Makefile.PL prevents INSTALLDIRS on command line.
+ RT #68287: Makefile.PL prevents INSTALLDIRS on command line.
+
+ * typo fix
+ RT #85335
+
+1.828 7 May 2013
+
+ * Minor change to build with Berkeley DB 6.x
+
+1.827 25 Jan 2012
+
+ * DB_File.pm - Don't use "@_" construct
+ [RT #79287]
+
+1.826 25 Jan 2012
+
+ * t/db-btree.t - fix use of "length @array"
+ [RT #74336]
+
+1.825 24 Jan 2012
+
+ * t/db-btree.t - fix use of "length @array"
+ [RT #74336]
+
+1.824 6 Aug 2011
+
+ * Amendments to tests to work in blead
+ [RT #70108]
+
+1.823 6 Aug 2011
+
+ * croak if attempt to freeze/thaw DB_File object
+ [RT #69985]
+
+1.822 12 March 2011
+
+ * Link rot
+ [rt.cpan.org #68739]
+
+1.822 12 March 2011
+
+ * Keep DB_File's warnings in sync with perl's
+ [rt.cpan.org #66339]
+
+1.821 10 January 2011
+
+ * Fixed typos & spelling errors.
+ [perl #81792]
+
+1.820 28 March 2009
+
+ * remove MAN3PODS from Makefile.PL to match core.
+
+1.819 18 February 2009
+
+ * t/db-recno.t fails if run in a path that contains spaces
+ [rt.cpan.org #43288]
+
+1.818 21 January 2009
+
+ * Updated Makefile.PL for Strawberry Perl.
+ Patch suggested by David Golden.
+
+ * Remove IRIX notes from README. The page referenced doesn't exist
+ anymore.
+
+1.817 27 March 2008
+
+ * Updated dbinfo
+
+ * Applied core patch 32299 - Re-apply change #30562
+
+ * Applied core patch 32208
+
+ * Applied core patch 32884 - use MM->parse_version() in Makefile.PL
+
+ * Applied core patch 32883 - Silence new warning grep in void context warning
+
+ * Applied core patch 32704 to remove use of PL_na in typemap
+
+ * Applied core patch 30562 to fix a build issue on OSF
+
+1.816 28 October 2007
+
+ * Clarified the warning about building with a different version of
+ Berkeley DB that is used at runtime.
+
+ * Also made the boot version check less strict.
+ [rt.cpan.org #30013]
+
+1.815 4 February 2007
+
+ * A few casting cleanups for building with C++ from Steve Peters.
+
+ * Fixed problem with recno which happened if you changed directory after
+ opening the database. Problem reported by Andrew Pam.
+
+
+1.814 11 November 2005
+
+ * Fix from Dominic Dunlop to tidy up an OS-X specific warning in
+ db-btree.t.
+
+ * Silenced a warning about $DB_File::Error only being used once.
+ Issue spotted by Dominic Dunlop.
+
+1.813 31st October 2005
+
+ * Updates for Berkeley DB 4.4
+
+1.812 9th October 2005
+
+ * Added libscan to Makefile.PL
+
+ * Fixed test failing under windows
+
+1.811 12th March 2005
+
+ * Fixed DBM filter bug in seq
+
+1.810 7th August 2004
+
+ * Fixed db-hash.t for Cygwin
+
+ * Added substr tests to db-hast.t
+
+ * Documented AIX build problem in README.
+
+1.809 20th June 2004
+
+ * Merged core patch 22258
+
+ * Merged core patch 22741
+
+ * Fixed core bug 30237.
+ Using substr to pass parameters to the low-level Berkeley DB interface
+ causes problems with Perl 5.8.1 or better.
+ typemap fix supplied by Marcus Holland-Moritz.
+
+1.808 22nd December 2003
+
+ * Added extra DBM Filter tests.
+
+ * Fixed a memory leak in ParseOpenInfo, which whould occur if the
+ opening of the database failed. Leak spotted by Adrian Enache.
+
+1.807 1st November 2003
+
+ * Fixed minor typos on pod documentation - reported by Jeremy Mates &
+ Mark Jason Dominus.
+
+ * dbinfo updated to report when a database is encrypted.
+
+1.806 22nd October 2002
+
+ * Fixed problem when trying to build with a multi-threaded perl.
+
+ * Tidied up the recursion detection code.
+
+ * merged core patch 17844 - missing dTHX declarations.
+
+ * merged core patch 17838
+
+1.805 1st September 2002
+
+ * Added support to allow DB_File to build with Berkeley DB 4.1.X
+
+ * Tightened up the test harness to test that calls to untie don't generate
+ the "untie attempted while %d inner references still exist" warning.
+
+ * added code to guard against calling the callbacks (compare,hash & prefix)
+ recursively.
+
+ * passing undef for the flags and/or mode when opening a database could cause
+ a "Use of uninitialized value in subroutine entry" warning. Now silenced.
+
+ * DBM filter code beefed up to cope with read-only $_.
+
+1.804 2nd June 2002
+
+ * Perl core patch 14939 added a new warning to "splice". This broke the
+ db-recno test harness. Fixed.
+
+ * merged core patches 16502 & 16540.
+
+1.803 1st March 2002
+
+ * Fixed a problem with db-btree.t where it complained about an "our"
+ variable redeclaration.
+
+ * FETCH, STORE & DELETE don't map the flags parameter into the
+ equivalent Berkeley DB function anymore.
+
+1.802 6th January 2002
+
+ * The message about some test failing in db-recno.t had the wrong test
+ numbers. Fixed.
+
+ * merged core patch 13942.
+
+1.801 26th November 2001
+
+ * Fixed typo in Makefile.PL
+
+ * Added "clean" attribute to Makefile.PL
+
+1.800 23rd November 2001
+
+ * use pport.h for perl backward compatibility code.
+
+ * use new ExtUtils::Constant module to generate XS constants.
+
+ * upgrade Makefile.PL upgrade/downgrade code to toggle "our" with
+ "use vars"
+
+1.79 22nd October 2001
+
+ * Added a "local $SIG{__DIE__}" inside the eval that checks for
+ the presence of XSLoader s suggested by Andrew Hryckowin.
+
+ * merged core patch 12277.
+
+ * Changed NEXTKEY to not initialise the input key. It isn't used anyway.
+
+1.79 22nd October 2001
+
+ * Fixed test harness for cygwin
+
+1.78 30th July 2001
+
+ * the test in Makefile.PL for AIX used -plthreads. Should have been
+ -lpthreads
+
+ * merged Core patches
+ 10372, 10335, 10372, 10534, 10549, 10643, 11051, 11194, 11432
+
+ * added documentation patch regarding duplicate keys from Andrew Johnson
+
+
+1.77 26th April 2001
+
+ * AIX is reported to need -lpthreads, so Makefile.PL now checks for
+ AIX and adds it to the link options.
+
+ * Minor documentation updates.
+
+ * Merged Core patch 9176
+
+ * Added a patch from Edward Avis that adds support for splice with
+ recno databases.
+
+ * Modified Makefile.PL to only enable the warnings pragma if using perl
+ 5.6.1 or better.
+
+1.76 15th January 2001
+
+ * Added instructions for using LD_PRELOAD to get Berkeley DB 2.x to work
+ with DB_File on Linux. Thanks to Norbert Bollow for sending details of
+ this approach.
+
+
+1.75 17th December 2000
+
+ * Fixed perl core patch 7703
+
+ * Added support to allow DB_File to be built with Berkeley DB 3.2 --
+ btree_compare, btree_prefix and hash_cb needed to be changed.
+
+ * Updated dbinfo to support Berkeley DB 3.2 file format changes.
+
+
+1.74 10th December 2000
+
+ * A "close" call in DB_File.xs needed parenthesised to stop win32 from
+ thinking it was one of its macros.
+
+ * Updated dbinfo to support Berkeley DB 3.1 file format changes.
+
+ * DB_File.pm & the test hasness now use the warnings pragma (when
+ available).
+
+ * Included Perl core patch 7703 -- size argument for hash_cb is different
+ for Berkeley DB 3.x
+
+ * Included Perl core patch 7801 -- Give __getBerkeleyDBInfo the ANSI C
+ treatment.
+
+ * @a = () produced the warning 'Argument "" isn't numeric in entersub'
+ This has been fixed. Thanks to Edward Avis for spotting this bug.
+
+ * Added note about building under Linux. Included patches.
+
+ * Included Perl core patch 8068 -- fix for bug 20001013.009
+ When run with warnings enabled "$hash{XX} = undef " produced an
+ "Uninitialized value" warning. This has been fixed.
+
+1.73 31st May 2000
+
+ * Added support in version.c for building with threaded Perl.
+
+ * Berkeley DB 3.1 has reenabled support for null keys. The test
+ harness has been updated to reflect this.
+
+1.72 16th January 2000
+
+ * Added hints/sco.pl
+
+ * The module will now use XSLoader when it is available. When it
+ isn't it will use DynaLoader.
+
+ * The locking section in DB_File.pm has been discredited. Many thanks
+ to David Harris for spotting the underlying problem, contributing
+ the updates to the documentation and writing DB_File::Lock (available
+ on CPAN).
+
+1.71 7th September 1999
+
+ * Fixed a bug that prevented 1.70 from compiling under win32
+
+ * Updated to support Berkeley DB 3.x
+
+ * Updated dbinfo for Berkeley DB 3.x file formats.
+
+1.70 4th August 1999
+
+ * Initialise $DB_File::db_ver and $DB_File::db_version with
+ GV_ADD|GV_ADDMULT -- bug spotted by Nick Ing-Simmons.
+
+ * Added a BOOT check to test for equivalent versions of db.h &
+ libdb.a/so.
+
+1.69 3rd August 1999
+
+ * fixed a bug in push -- DB_APPEND wasn't working properly.
+
+ * Fixed the R_SETCURSOR bug introduced in 1.68
+
+ * Added a new Perl variable $DB_File::db_ver
+
+1.68 22nd July 1999
+
+ * Merged changes from 5.005_58
+
+ * Fixed a bug in R_IBEFORE & R_IAFTER processing in Berkeley DB
+ 2 databases.
+
+ * Added some of the examples in the POD into the test harness.
+
+1.67 6th June 1999
+
+ * Added DBM Filter documentation to DB_File.pm
+
+ * Fixed DBM Filter code to work with 5.004
+
+ * A few instances of newSVpvn were used in 1.66. This isn't available in
+ Perl 5.004_04 or earlier. Replaced with newSVpv.
+
+1.66 15th March 1999
+
+ * Added DBM Filter code
+
+1.65 6th March 1999
+
+ * Fixed a bug in the recno PUSH logic.
+ * The BOOT version check now needs 2.3.4 when using Berkeley DB version 2
+
+1.64 21st February 1999
+
+ * Tidied the 1.x to 2.x flag mapping code.
+ * Added a patch from Mark Kettenis to fix a flag
+ mapping problem with O_RDONLY on the Hurd
+ * Updated the message that db-recno.t prints when tests 51, 53 or 55 fail.
+
+1.63 19th December 1998
+
+ * Fix to allow DB 2.6.x to build with DB_File
+ * Documentation updated to use push,pop etc in the RECNO example &
+ to include the find_dup & del_dup methods.
+
+1.62 30th November 1998
+
+ Added hints/dynixptx.pl.
+ Fixed typemap -- 1.61 used PL_na instead of na
+
+1.61 19th November 1998
+
+ Added a note to README about how to build Berkeley DB 2.x when
+ using HP-UX.
+ Minor modifications to get the module to build with DB 2.5.x
+ Fixed a typo in the definition of O_RDONLY, courtesy of Mark Kettenis.
+
+1.60
+ Changed the test to check for full tied array support
+
+1.59
+ Updated the license section.
+
+ Berkeley DB 2.4.10 disallows zero length keys. Tests 32 & 42 in
+ db-btree.t and test 27 in db-hash.t failed because of this change.
+ Those tests have been zapped.
+
+ Added dbinfo to the distribution.
+
+1.58
+ Tied Array support was enhanced in Perl 5.004_57. DB_File now
+ supports PUSH,POP,SHIFT,UNSHIFT & STORESIZE.
+
+ Fixed a problem with the use of sv_setpvn. When the size is
+ specified as 0, it does a strlen on the data. This was ok for DB
+ 1.x, but isn't for DB 2.x.
+
+1.57
+ If Perl has been compiled with Threads support,the symbol op will be
+ defined. This clashes with a field name in db.h, so it needs to be
+ #undef'ed before db.h is included.
+
+1.56
+ Documented the Solaris 2.5 mutex bug
+
+1.55
+ Merged 1.16 changes.
+
+1.54
+
+ Fixed a small bug in the test harness when run under win32
+ The emulation of fd when useing DB 2.x was busted.
+
+1.53
+
+ Added DB_RENUMBER to flags for recno.
+
+1.52
+
+ Patch from Nick Ing-Simmons now allows DB_File to build on NT.
+ Merged 1.15 patch.
+
+1.51
+
+ Fixed the test harness so that it doesn't expect DB_File to have
+ been installed by the main Perl build.
+
+
+ Fixed a bug in mapping 1.x O_RDONLY flag to 2.x DB_RDONLY equivalent
+
+1.50
+
+ DB_File can now build with either DB 1.x or 2.x, but not both at
+ the same time.
+
+1.16
+
+ A harmless looking tab was causing Makefile.PL to fail on AIX 3.2.5
+
+ Small fix for the AIX strict C compiler XLC which doesn't like
+ __attribute__ being defined via proto.h and redefined via db.h. Fix
+ courtesy of Jarkko Hietaniemi.
+
+1.15
+
+ Patch from Gisle Aas to suppress "use of undefined
+ value" warning with db_get and db_seq.
+
+ Patch from Gisle Aas to make DB_File export only the
+ O_* constants from Fcntl.
+
+ Removed the DESTROY method from the DB_File::HASHINFO module.
+
+ Previously DB_File hard-wired the class name of any object that it
+ created to "DB_File". This makes sub-classing difficult. Now
+ DB_File creats objects in the namespace of the package it has been
+ inherited into.
+
+
+1.14
+
+ Made it illegal to tie an associative array to a RECNO database and
+ an ordinary array to a HASH or BTREE database.
+
+1.13
+
+ Minor changes to DB_FIle.xs and DB_File.pm
+
+1.12
+
+ Documented the incompatibility with version 2 of Berkeley DB.
+
+1.11
+
+ Documented the untie gotcha.
+
+1.10
+
+ Fixed fd method so that it still returns -1 for in-memory files
+ when db 1.86 is used.
+
+1.09
+
+ Minor bug fix in DB_File::HASHINFO, DB_File::RECNOINFO and
+ DB_File::BTREEINFO.
+
+ Changed default mode to 0666.
+
+1.08
+
+ Documented operation of bval.
+
+1.07
+
+ Fixed bug with RECNO, where bval wasn't defaulting to "\n".
+
+1.06
+
+ Minor namespace cleanup: Localized PrintBtree.
+
+1.05
+
+ Made all scripts in the documentation strict and -w clean.
+
+ Added logic to DB_File.xs to allow the module to be built after
+ Perl is installed.
+
+1.04
+
+ Minor documentation changes.
+
+ Fixed a bug in hash_cb. Patches supplied by Dave Hammen,
+ .
+
+ Fixed a bug with the constructors for DB_File::HASHINFO,
+ DB_File::BTREEINFO and DB_File::RECNOINFO. Also tidied up the
+ constructors to make them -w clean.
+
+ Reworked part of the test harness to be more locale friendly.
+
+1.03
+
+ Documentation update.
+
+ DB_File now imports the constants (O_RDWR, O_CREAT etc.) from Fcntl
+ automatically.
+
+ The standard hash function exists is now supported.
+
+ Modified the behavior of get_dup. When it returns an associative
+ array, the value is the count of the number of matching BTREE
+ values.
+
+1.02
+
+ Merged OS/2 specific code into DB_File.xs
+
+ Removed some redundant code in DB_File.xs.
+
+ Documentation update.
+
+ Allow negative subscripts with RECNO interface.
+
+ Changed the default flags from O_RDWR to O_CREAT|O_RDWR.
+
+ The example code which showed how to lock a database needed a call
+ to sync added. Without it the resultant database file was empty.
+
+ Added get_dup method.
+
+1.01
+
+ Fixed a core dump problem with SunOS.
+
+ The return value from TIEHASH wasn't set to NULL when dbopen
+ returned an error.
+
+1.0
+
+ DB_File has been in use for over a year. To reflect that, the
+ version number has been incremented to 1.0.
+
+ Added complete support for multiple concurrent callbacks.
+
+ Using the push method on an empty list didn't work properly. This
+ has been fixed.
+
+0.3
+
+ Added prototype support for multiple btree compare callbacks.
+
+0.2
+
+ When DB_File is opening a database file it no longer terminates the
+ process if dbopen returned an error. This allows file protection
+ errors to be caught at run time. Thanks to Judith Grass
+ for spotting the bug.
+
+0.1
+
+ First Release.
+
diff --git a/fastSum/resources/ROUGE/DB_File-1.835/DB_File.bs b/fastSum/resources/ROUGE/DB_File-1.835/DB_File.bs
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/fastSum/resources/ROUGE/DB_File-1.835/DB_File.c b/fastSum/resources/ROUGE/DB_File-1.835/DB_File.c
new file mode 100644
index 0000000000000000000000000000000000000000..45cec903debad349c2a5d68d9aebf6f192fc68eb
--- /dev/null
+++ b/fastSum/resources/ROUGE/DB_File-1.835/DB_File.c
@@ -0,0 +1,3152 @@
+/*
+ * This file was generated automatically by ExtUtils::ParseXS version 3.28 from the
+ * contents of DB_File.xs. Do not edit this file, edit DB_File.xs instead.
+ *
+ * ANY CHANGES MADE HERE WILL BE LOST!
+ *
+ */
+
+#line 1 "DB_File.xs"
+/*
+
+ DB_File.xs -- Perl 5 interface to Berkeley DB
+
+ Written by Paul Marquess
+
+ All comments/suggestions/problems are welcome
+
+ Copyright (c) 1995-2014 Paul Marquess. All rights reserved.
+ This program is free software; you can redistribute it and/or
+ modify it under the same terms as Perl itself.
+
+ Changes:
+ 0.1 - Initial Release
+ 0.2 - No longer bombs out if dbopen returns an error.
+ 0.3 - Added some support for multiple btree compares
+ 1.0 - Complete support for multiple callbacks added.
+ Fixed a problem with pushing a value onto an empty list.
+ 1.01 - Fixed a SunOS core dump problem.
+ The return value from TIEHASH wasn't set to NULL when
+ dbopen returned an error.
+ 1.02 - Use ALIAS to define TIEARRAY.
+ Removed some redundant commented code.
+ Merged OS2 code into the main distribution.
+ Allow negative subscripts with RECNO interface.
+ Changed the default flags to O_CREAT|O_RDWR
+ 1.03 - Added EXISTS
+ 1.04 - fixed a couple of bugs in hash_cb. Patches supplied by
+ Dave Hammen, hammen@gothamcity.jsc.nasa.gov
+ 1.05 - Added logic to allow prefix & hash types to be specified via
+ Makefile.PL
+ 1.06 - Minor namespace cleanup: Localized PrintBtree.
+ 1.07 - Fixed bug with RECNO, where bval wasn't defaulting to "\n".
+ 1.08 - No change to DB_File.xs
+ 1.09 - Default mode for dbopen changed to 0666
+ 1.10 - Fixed fd method so that it still returns -1 for
+ in-memory files when db 1.86 is used.
+ 1.11 - No change to DB_File.xs
+ 1.12 - No change to DB_File.xs
+ 1.13 - Tidied up a few casts.
+ 1.14 - Made it illegal to tie an associative array to a RECNO
+ database and an ordinary array to a HASH or BTREE database.
+ 1.50 - Make work with both DB 1.x or DB 2.x
+ 1.51 - Fixed a bug in mapping 1.x O_RDONLY flag to 2.x DB_RDONLY equivalent
+ 1.52 - Patch from Gisle Aas to suppress "use of
+ undefined value" warning with db_get and db_seq.
+ 1.53 - Added DB_RENUMBER to flags for recno.
+ 1.54 - Fixed bug in the fd method
+ 1.55 - Fix for AIX from Jarkko Hietaniemi
+ 1.56 - No change to DB_File.xs
+ 1.57 - added the #undef op to allow building with Threads support.
+ 1.58 - Fixed a problem with the use of sv_setpvn. When the
+ size is specified as 0, it does a strlen on the data.
+ This was ok for DB 1.x, but isn't for DB 2.x.
+ 1.59 - No change to DB_File.xs
+ 1.60 - Some code tidy up
+ 1.61 - added flagSet macro for DB 2.5.x
+ fixed typo in O_RDONLY test.
+ 1.62 - No change to DB_File.xs
+ 1.63 - Fix to alllow DB 2.6.x to build.
+ 1.64 - Tidied up the 1.x to 2.x flags mapping code.
+ Added a patch from Mark Kettenis
+ to fix a flag mapping problem with O_RDONLY on the Hurd
+ 1.65 - Fixed a bug in the PUSH logic.
+ Added BOOT check that using 2.3.4 or greater
+ 1.66 - Added DBM filter code
+ 1.67 - Backed off the use of newSVpvn.
+ Fixed DBM Filter code for Perl 5.004.
+ Fixed a small memory leak in the filter code.
+ 1.68 - fixed backward compatibility bug with R_IAFTER & R_IBEFORE
+ merged in the 5.005_58 changes
+ 1.69 - fixed a bug in push -- DB_APPEND wasn't working properly.
+ Fixed the R_SETCURSOR bug introduced in 1.68
+ Added a new Perl variable $DB_File::db_ver
+ 1.70 - Initialise $DB_File::db_ver and $DB_File::db_version with
+ GV_ADD|GV_ADDMULT -- bug spotted by Nick Ing-Simmons.
+ Added a BOOT check to test for equivalent versions of db.h &
+ libdb.a/so.
+ 1.71 - Support for Berkeley DB version 3.
+ Support for Berkeley DB 2/3's backward compatibility mode.
+ Rewrote push
+ 1.72 - No change to DB_File.xs
+ 1.73 - No change to DB_File.xs
+ 1.74 - A call to open needed parenthesised to stop it clashing
+ with a win32 macro.
+ Added Perl core patches 7703 & 7801.
+ 1.75 - Fixed Perl core patch 7703.
+ Added support to allow DB_File to be built with
+ Berkeley DB 3.2 -- btree_compare, btree_prefix and hash_cb
+ needed to be changed.
+ 1.76 - No change to DB_File.xs
+ 1.77 - Tidied up a few types used in calling newSVpvn.
+ 1.78 - Core patch 10335, 10372, 10534, 10549, 11051 included.
+ 1.79 - NEXTKEY ignores the input key.
+ Added lots of casts
+ 1.800 - Moved backward compatibility code into ppport.h.
+ Use the new constants code.
+ 1.801 - No change to DB_File.xs
+ 1.802 - No change to DB_File.xs
+ 1.803 - FETCH, STORE & DELETE don't map the flags parameter
+ into the equivalent Berkeley DB function anymore.
+ 1.804 - no change.
+ 1.805 - recursion detection added to the callbacks
+ Support for 4.1.X added.
+ Filter code can now cope with read-only $_
+ 1.806 - recursion detection beefed up.
+ 1.807 - no change
+ 1.808 - leak fixed in ParseOpenInfo
+ 1.809 - no change
+ 1.810 - no change
+ 1.811 - no change
+ 1.812 - no change
+ 1.813 - no change
+ 1.814 - no change
+ 1.814 - C++ casting fixes
+
+*/
+
+#define PERL_NO_GET_CONTEXT
+#include "EXTERN.h"
+#include "perl.h"
+#include "XSUB.h"
+
+#ifdef _NOT_CORE
+# include "ppport.h"
+#endif
+
+int DB_File___unused() { return 0; }
+
+/* Mention DB_VERSION_MAJOR_CFG, DB_VERSION_MINOR_CFG, and
+ DB_VERSION_PATCH_CFG here so that Configure pulls them all in. */
+
+/* Being the Berkeley DB we prefer the (which will be
+ * shortly #included by the ) __attribute__ to the possibly
+ * already defined __attribute__, for example by GNUC or by Perl. */
+
+/* #if DB_VERSION_MAJOR_CFG < 2 */
+#ifndef DB_VERSION_MAJOR
+# undef __attribute__
+#endif
+
+#ifdef COMPAT185
+# include
+#else
+
+/* Uncomment one of the lines below */
+/* See the section "At least one secondary cursor must be specified to DB->join"
+ in the README file for the circumstances where you need to uncomment one
+ of the two lines below.
+*/
+
+/* #define time_t __time64_t */
+/* #define time_t __time32_t */
+
+# include
+#endif
+
+#ifndef PERL_UNUSED_ARG
+# define PERL_UNUSED_ARG(x) ((void)x)
+#endif
+
+/* Wall starts with 5.7.x */
+
+#if PERL_REVISION > 5 || (PERL_REVISION == 5 && PERL_VERSION >= 7)
+
+/* Since we dropped the gccish definition of __attribute__ we will want
+ * to redefine dNOOP, however (so that dTHX continues to work). Yes,
+ * all this means that we can't do attribute checking on the DB_File,
+ * boo, hiss. */
+# ifndef DB_VERSION_MAJOR
+
+# undef dNOOP
+# ifdef __cplusplus
+# define dNOOP (void)0
+# else
+# define dNOOP extern int DB_File___notused()
+# endif
+
+ /* Ditto for dXSARGS. */
+# undef dXSARGS
+# define dXSARGS \
+ dSP; dMARK; \
+ I32 ax = mark - PL_stack_base + 1; \
+ I32 items = sp - mark
+
+# endif
+
+/* avoid -Wall; DB_File xsubs never make use of `ix' setup for ALIASes */
+# undef dXSI32
+# define dXSI32 dNOOP
+
+#endif /* Perl >= 5.7 */
+
+#include
+
+/* #define TRACE */
+
+#ifdef TRACE
+# define Trace(x) printf x
+#else
+# define Trace(x)
+#endif
+
+
+#define DBT_clear(x) Zero(&x, 1, DBT) ;
+
+#ifdef DB_VERSION_MAJOR
+
+#if DB_VERSION_MAJOR == 2
+# define BERKELEY_DB_1_OR_2
+#endif
+
+#if DB_VERSION_MAJOR > 3 || (DB_VERSION_MAJOR == 3 && DB_VERSION_MINOR >= 2)
+# define AT_LEAST_DB_3_2
+#endif
+
+#if DB_VERSION_MAJOR > 3 || (DB_VERSION_MAJOR == 3 && DB_VERSION_MINOR >= 3)
+# define AT_LEAST_DB_3_3
+#endif
+
+#if DB_VERSION_MAJOR > 4 || (DB_VERSION_MAJOR == 4 && DB_VERSION_MINOR >= 1)
+# define AT_LEAST_DB_4_1
+#endif
+
+#if DB_VERSION_MAJOR > 4 || (DB_VERSION_MAJOR == 4 && DB_VERSION_MINOR >= 3)
+# define AT_LEAST_DB_4_3
+#endif
+
+#if DB_VERSION_MAJOR >= 6
+# define AT_LEAST_DB_6_0
+#endif
+
+#ifdef AT_LEAST_DB_3_3
+# define WANT_ERROR
+#endif
+
+/* map version 2 features & constants onto their version 1 equivalent */
+
+#ifdef DB_Prefix_t
+# undef DB_Prefix_t
+#endif
+#define DB_Prefix_t size_t
+
+#ifdef DB_Hash_t
+# undef DB_Hash_t
+#endif
+#define DB_Hash_t u_int32_t
+
+/* DBTYPE stays the same */
+/* HASHINFO, RECNOINFO and BTREEINFO map to DB_INFO */
+#if DB_VERSION_MAJOR == 2
+ typedef DB_INFO INFO ;
+#else /* DB_VERSION_MAJOR > 2 */
+# define DB_FIXEDLEN (0x8000)
+#endif /* DB_VERSION_MAJOR == 2 */
+
+/* version 2 has db_recno_t in place of recno_t */
+typedef db_recno_t recno_t;
+
+
+#define R_CURSOR DB_SET_RANGE
+#define R_FIRST DB_FIRST
+#define R_IAFTER DB_AFTER
+#define R_IBEFORE DB_BEFORE
+#define R_LAST DB_LAST
+#define R_NEXT DB_NEXT
+#define R_NOOVERWRITE DB_NOOVERWRITE
+#define R_PREV DB_PREV
+
+#if DB_VERSION_MAJOR == 2 && DB_VERSION_MINOR < 5
+# define R_SETCURSOR 0x800000
+#else
+# define R_SETCURSOR (-100)
+#endif
+
+#define R_RECNOSYNC 0
+#define R_FIXEDLEN DB_FIXEDLEN
+#define R_DUP DB_DUP
+
+
+#define db_HA_hash h_hash
+#define db_HA_ffactor h_ffactor
+#define db_HA_nelem h_nelem
+#define db_HA_bsize db_pagesize
+#define db_HA_cachesize db_cachesize
+#define db_HA_lorder db_lorder
+
+#define db_BT_compare bt_compare
+#define db_BT_prefix bt_prefix
+#define db_BT_flags flags
+#define db_BT_psize db_pagesize
+#define db_BT_cachesize db_cachesize
+#define db_BT_lorder db_lorder
+#define db_BT_maxkeypage
+#define db_BT_minkeypage
+
+
+#define db_RE_reclen re_len
+#define db_RE_flags flags
+#define db_RE_bval re_pad
+#define db_RE_bfname re_source
+#define db_RE_psize db_pagesize
+#define db_RE_cachesize db_cachesize
+#define db_RE_lorder db_lorder
+
+#define TXN NULL,
+
+#define do_SEQ(db, key, value, flag) (db->cursor->c_get)(db->cursor, &key, &value, flag)
+
+
+#define DBT_flags(x) x.flags = 0
+#define DB_flags(x, v) x |= v
+
+#if DB_VERSION_MAJOR == 2 && DB_VERSION_MINOR < 5
+# define flagSet(flags, bitmask) ((flags) & (bitmask))
+#else
+# define flagSet(flags, bitmask) (((flags) & DB_OPFLAGS_MASK) == (u_int)(bitmask))
+#endif
+
+#else /* db version 1.x */
+
+#define BERKELEY_DB_1
+#define BERKELEY_DB_1_OR_2
+
+typedef union INFO {
+ HASHINFO hash ;
+ RECNOINFO recno ;
+ BTREEINFO btree ;
+ } INFO ;
+
+
+#ifdef mDB_Prefix_t
+# ifdef DB_Prefix_t
+# undef DB_Prefix_t
+# endif
+# define DB_Prefix_t mDB_Prefix_t
+#endif
+
+#ifdef mDB_Hash_t
+# ifdef DB_Hash_t
+# undef DB_Hash_t
+# endif
+# define DB_Hash_t mDB_Hash_t
+#endif
+
+#define db_HA_hash hash.hash
+#define db_HA_ffactor hash.ffactor
+#define db_HA_nelem hash.nelem
+#define db_HA_bsize hash.bsize
+#define db_HA_cachesize hash.cachesize
+#define db_HA_lorder hash.lorder
+
+#define db_BT_compare btree.compare
+#define db_BT_prefix btree.prefix
+#define db_BT_flags btree.flags
+#define db_BT_psize btree.psize
+#define db_BT_cachesize btree.cachesize
+#define db_BT_lorder btree.lorder
+#define db_BT_maxkeypage btree.maxkeypage
+#define db_BT_minkeypage btree.minkeypage
+
+#define db_RE_reclen recno.reclen
+#define db_RE_flags recno.flags
+#define db_RE_bval recno.bval
+#define db_RE_bfname recno.bfname
+#define db_RE_psize recno.psize
+#define db_RE_cachesize recno.cachesize
+#define db_RE_lorder recno.lorder
+
+#define TXN
+
+#define do_SEQ(db, key, value, flag) (db->dbp->seq)(db->dbp, &key, &value, flag)
+#define DBT_flags(x)
+#define DB_flags(x, v)
+#define flagSet(flags, bitmask) ((flags) & (bitmask))
+
+#endif /* db version 1 */
+
+
+
+#define db_DELETE(db, key, flags) ((db->dbp)->del)(db->dbp, TXN &key, 0)
+#define db_STORE(db, key, value, flags) ((db->dbp)->put)(db->dbp, TXN &key, &value, 0)
+#define db_FETCH(db, key, flags) ((db->dbp)->get)(db->dbp, TXN &key, &value, 0)
+
+#define db_sync(db, flags) ((db->dbp)->sync)(db->dbp, flags)
+#define db_get(db, key, value, flags) ((db->dbp)->get)(db->dbp, TXN &key, &value, flags)
+
+#ifdef DB_VERSION_MAJOR
+#define db_DESTROY(db) (!db->aborted && ( db->cursor->c_close(db->cursor),\
+ (db->dbp->close)(db->dbp, 0) ))
+#define db_close(db) ((db->dbp)->close)(db->dbp, 0)
+#define db_del(db, key, flags) (flagSet(flags, R_CURSOR) \
+ ? ((db->cursor)->c_del)(db->cursor, 0) \
+ : ((db->dbp)->del)(db->dbp, NULL, &key, flags) )
+
+#else /* ! DB_VERSION_MAJOR */
+
+#define db_DESTROY(db) (!db->aborted && ((db->dbp)->close)(db->dbp))
+#define db_close(db) ((db->dbp)->close)(db->dbp)
+#define db_del(db, key, flags) ((db->dbp)->del)(db->dbp, &key, flags)
+#define db_put(db, key, value, flags) ((db->dbp)->put)(db->dbp, &key, &value, flags)
+
+#endif /* ! DB_VERSION_MAJOR */
+
+
+#define db_seq(db, key, value, flags) do_SEQ(db, key, value, flags)
+
+typedef struct {
+ DBTYPE type ;
+ DB * dbp ;
+ SV * compare ;
+ bool in_compare ;
+ SV * prefix ;
+ bool in_prefix ;
+ SV * hash ;
+ bool in_hash ;
+ bool aborted ;
+ int in_memory ;
+#ifdef BERKELEY_DB_1_OR_2
+ INFO info ;
+#endif
+#ifdef DB_VERSION_MAJOR
+ DBC * cursor ;
+#endif
+ SV * filter_fetch_key ;
+ SV * filter_store_key ;
+ SV * filter_fetch_value ;
+ SV * filter_store_value ;
+ int filtering ;
+
+ } DB_File_type;
+
+typedef DB_File_type * DB_File ;
+typedef DBT DBTKEY ;
+
+#define my_sv_setpvn(sv, d, s) sv_setpvn(sv, (s ? d : (const char *)""), s)
+
+#define OutputValue(arg, name) \
+ { if (RETVAL == 0) { \
+ SvGETMAGIC(arg) ; \
+ my_sv_setpvn(arg, (const char *)name.data, name.size) ; \
+ TAINT; \
+ SvTAINTED_on(arg); \
+ SvUTF8_off(arg); \
+ DBM_ckFilter(arg, filter_fetch_value,"filter_fetch_value") ; \
+ } \
+ }
+
+#define OutputKey(arg, name) \
+ { if (RETVAL == 0) \
+ { \
+ SvGETMAGIC(arg) ; \
+ if (db->type != DB_RECNO) { \
+ my_sv_setpvn(arg, (const char *)name.data, name.size); \
+ } \
+ else \
+ sv_setiv(arg, (I32)*(I32*)name.data - 1); \
+ TAINT; \
+ SvTAINTED_on(arg); \
+ SvUTF8_off(arg); \
+ DBM_ckFilter(arg, filter_fetch_key,"filter_fetch_key") ; \
+ } \
+ }
+
+#define my_SvUV32(sv) ((u_int32_t)SvUV(sv))
+
+#ifdef CAN_PROTOTYPE
+extern void __getBerkeleyDBInfo(void);
+#endif
+
+/* Internal Global Data */
+
+#define MY_CXT_KEY "DB_File::_guts" XS_VERSION
+
+typedef struct {
+ recno_t x_Value;
+ recno_t x_zero;
+ DB_File x_CurrentDB;
+ DBTKEY x_empty;
+} my_cxt_t;
+
+START_MY_CXT
+
+#define Value (MY_CXT.x_Value)
+#define zero (MY_CXT.x_zero)
+#define CurrentDB (MY_CXT.x_CurrentDB)
+#define empty (MY_CXT.x_empty)
+
+#define ERR_BUFF "DB_File::Error"
+
+#ifdef DB_VERSION_MAJOR
+
+static int
+#ifdef CAN_PROTOTYPE
+db_put(DB_File db, DBTKEY key, DBT value, u_int flags)
+#else
+db_put(db, key, value, flags)
+DB_File db ;
+DBTKEY key ;
+DBT value ;
+u_int flags ;
+#endif
+{
+ int status ;
+
+ if (flagSet(flags, R_IAFTER) || flagSet(flags, R_IBEFORE)) {
+ DBC * temp_cursor ;
+ DBT l_key, l_value;
+
+#if DB_VERSION_MAJOR == 2 && DB_VERSION_MINOR < 6
+ if (((db->dbp)->cursor)(db->dbp, NULL, &temp_cursor) != 0)
+#else
+ if (((db->dbp)->cursor)(db->dbp, NULL, &temp_cursor, 0) != 0)
+#endif
+ return (-1) ;
+
+ memset(&l_key, 0, sizeof(l_key));
+ l_key.data = key.data;
+ l_key.size = key.size;
+ memset(&l_value, 0, sizeof(l_value));
+ l_value.data = value.data;
+ l_value.size = value.size;
+
+ if ( temp_cursor->c_get(temp_cursor, &l_key, &l_value, DB_SET) != 0) {
+ (void)temp_cursor->c_close(temp_cursor);
+ return (-1);
+ }
+
+ status = temp_cursor->c_put(temp_cursor, &key, &value, flags);
+ (void)temp_cursor->c_close(temp_cursor);
+
+ return (status) ;
+ }
+
+
+ if (flagSet(flags, R_CURSOR)) {
+ return ((db->cursor)->c_put)(db->cursor, &key, &value, DB_CURRENT);
+ }
+
+ if (flagSet(flags, R_SETCURSOR)) {
+ if ((db->dbp)->put(db->dbp, NULL, &key, &value, 0) != 0)
+ return -1 ;
+ return ((db->cursor)->c_get)(db->cursor, &key, &value, DB_SET_RANGE);
+
+ }
+
+ return ((db->dbp)->put)(db->dbp, NULL, &key, &value, flags) ;
+
+}
+
+#endif /* DB_VERSION_MAJOR */
+
+static void
+tidyUp(DB_File db)
+{
+ db->aborted = TRUE ;
+}
+
+
+static int
+
+#ifdef AT_LEAST_DB_6_0
+#ifdef CAN_PROTOTYPE
+btree_compare(DB * db, const DBT *key1, const DBT *key2, size_t* locp)
+#else
+btree_compare(db, key1, key2, locp)
+DB * db ;
+const DBT * key1 ;
+const DBT * key2 ;
+size_t* locp;
+#endif /* CAN_PROTOTYPE */
+
+#else /* Berkeley DB < 6.0 */
+#ifdef AT_LEAST_DB_3_2
+
+#ifdef CAN_PROTOTYPE
+btree_compare(DB * db, const DBT *key1, const DBT *key2)
+#else
+btree_compare(db, key1, key2)
+DB * db ;
+const DBT * key1 ;
+const DBT * key2 ;
+#endif /* CAN_PROTOTYPE */
+
+#else /* Berkeley DB < 3.2 */
+
+#ifdef CAN_PROTOTYPE
+btree_compare(const DBT *key1, const DBT *key2)
+#else
+btree_compare(key1, key2)
+const DBT * key1 ;
+const DBT * key2 ;
+#endif
+
+#endif
+#endif
+
+{
+#ifdef dTHX
+ dTHX;
+#endif
+ dSP ;
+ dMY_CXT ;
+ void * data1, * data2 ;
+ int retval ;
+ int count ;
+
+#ifdef AT_LEAST_DB_3_2
+ PERL_UNUSED_ARG(db);
+#endif
+
+ if (CurrentDB->in_compare) {
+ tidyUp(CurrentDB);
+ croak ("DB_File btree_compare: recursion detected\n") ;
+ }
+
+ data1 = (char *) key1->data ;
+ data2 = (char *) key2->data ;
+
+#ifndef newSVpvn
+ /* As newSVpv will assume that the data pointer is a null terminated C
+ string if the size parameter is 0, make sure that data points to an
+ empty string if the length is 0
+ */
+ if (key1->size == 0)
+ data1 = "" ;
+ if (key2->size == 0)
+ data2 = "" ;
+#endif
+
+ ENTER ;
+ SAVETMPS;
+ SAVESPTR(CurrentDB);
+ CurrentDB->in_compare = FALSE;
+ SAVEINT(CurrentDB->in_compare);
+ CurrentDB->in_compare = TRUE;
+
+ PUSHMARK(SP) ;
+ EXTEND(SP,2) ;
+ PUSHs(sv_2mortal(newSVpvn((const char*)data1,key1->size)));
+ PUSHs(sv_2mortal(newSVpvn((const char*)data2,key2->size)));
+ PUTBACK ;
+
+ count = perl_call_sv(CurrentDB->compare, G_SCALAR);
+
+ SPAGAIN ;
+
+ if (count != 1){
+ tidyUp(CurrentDB);
+ croak ("DB_File btree_compare: expected 1 return value from compare sub, got %d\n", count) ;
+ }
+
+ retval = POPi ;
+
+ PUTBACK ;
+ FREETMPS ;
+ LEAVE ;
+
+ return (retval) ;
+
+}
+
+static DB_Prefix_t
+#ifdef AT_LEAST_DB_3_2
+
+#ifdef CAN_PROTOTYPE
+btree_prefix(DB * db, const DBT *key1, const DBT *key2)
+#else
+btree_prefix(db, key1, key2)
+Db * db ;
+const DBT * key1 ;
+const DBT * key2 ;
+#endif
+
+#else /* Berkeley DB < 3.2 */
+
+#ifdef CAN_PROTOTYPE
+btree_prefix(const DBT *key1, const DBT *key2)
+#else
+btree_prefix(key1, key2)
+const DBT * key1 ;
+const DBT * key2 ;
+#endif
+
+#endif
+{
+#ifdef dTHX
+ dTHX;
+#endif
+ dSP ;
+ dMY_CXT ;
+ char * data1, * data2 ;
+ int retval ;
+ int count ;
+
+#ifdef AT_LEAST_DB_3_2
+ PERL_UNUSED_ARG(db);
+#endif
+
+ if (CurrentDB->in_prefix){
+ tidyUp(CurrentDB);
+ croak ("DB_File btree_prefix: recursion detected\n") ;
+ }
+
+ data1 = (char *) key1->data ;
+ data2 = (char *) key2->data ;
+
+#ifndef newSVpvn
+ /* As newSVpv will assume that the data pointer is a null terminated C
+ string if the size parameter is 0, make sure that data points to an
+ empty string if the length is 0
+ */
+ if (key1->size == 0)
+ data1 = "" ;
+ if (key2->size == 0)
+ data2 = "" ;
+#endif
+
+ ENTER ;
+ SAVETMPS;
+ SAVESPTR(CurrentDB);
+ CurrentDB->in_prefix = FALSE;
+ SAVEINT(CurrentDB->in_prefix);
+ CurrentDB->in_prefix = TRUE;
+
+ PUSHMARK(SP) ;
+ EXTEND(SP,2) ;
+ PUSHs(sv_2mortal(newSVpvn(data1,key1->size)));
+ PUSHs(sv_2mortal(newSVpvn(data2,key2->size)));
+ PUTBACK ;
+
+ count = perl_call_sv(CurrentDB->prefix, G_SCALAR);
+
+ SPAGAIN ;
+
+ if (count != 1){
+ tidyUp(CurrentDB);
+ croak ("DB_File btree_prefix: expected 1 return value from prefix sub, got %d\n", count) ;
+ }
+
+ retval = POPi ;
+
+ PUTBACK ;
+ FREETMPS ;
+ LEAVE ;
+
+ return (retval) ;
+}
+
+
+#ifdef BERKELEY_DB_1
+# define HASH_CB_SIZE_TYPE size_t
+#else
+# define HASH_CB_SIZE_TYPE u_int32_t
+#endif
+
+static DB_Hash_t
+#ifdef AT_LEAST_DB_3_2
+
+#ifdef CAN_PROTOTYPE
+hash_cb(DB * db, const void *data, u_int32_t size)
+#else
+hash_cb(db, data, size)
+DB * db ;
+const void * data ;
+HASH_CB_SIZE_TYPE size ;
+#endif
+
+#else /* Berkeley DB < 3.2 */
+
+#ifdef CAN_PROTOTYPE
+hash_cb(const void *data, HASH_CB_SIZE_TYPE size)
+#else
+hash_cb(data, size)
+const void * data ;
+HASH_CB_SIZE_TYPE size ;
+#endif
+
+#endif
+{
+#ifdef dTHX
+ dTHX;
+#endif
+ dSP ;
+ dMY_CXT;
+ int retval = 0;
+ int count ;
+
+#ifdef AT_LEAST_DB_3_2
+ PERL_UNUSED_ARG(db);
+#endif
+
+ if (CurrentDB->in_hash){
+ tidyUp(CurrentDB);
+ croak ("DB_File hash callback: recursion detected\n") ;
+ }
+
+#ifndef newSVpvn
+ if (size == 0)
+ data = "" ;
+#endif
+
+ /* DGH - Next two lines added to fix corrupted stack problem */
+ ENTER ;
+ SAVETMPS;
+ SAVESPTR(CurrentDB);
+ CurrentDB->in_hash = FALSE;
+ SAVEINT(CurrentDB->in_hash);
+ CurrentDB->in_hash = TRUE;
+
+ PUSHMARK(SP) ;
+
+
+ XPUSHs(sv_2mortal(newSVpvn((char*)data,size)));
+ PUTBACK ;
+
+ count = perl_call_sv(CurrentDB->hash, G_SCALAR);
+
+ SPAGAIN ;
+
+ if (count != 1){
+ tidyUp(CurrentDB);
+ croak ("DB_File hash_cb: expected 1 return value from hash sub, got %d\n", count) ;
+ }
+
+ retval = POPi ;
+
+ PUTBACK ;
+ FREETMPS ;
+ LEAVE ;
+
+ return (retval) ;
+}
+
+#ifdef WANT_ERROR
+
+static void
+#ifdef AT_LEAST_DB_4_3
+db_errcall_cb(const DB_ENV* dbenv, const char * db_errpfx, const char * buffer)
+#else
+db_errcall_cb(const char * db_errpfx, char * buffer)
+#endif
+{
+#ifdef dTHX
+ dTHX;
+#endif
+ SV * sv = perl_get_sv(ERR_BUFF, FALSE) ;
+#ifdef AT_LEAST_DB_4_3
+ PERL_UNUSED_ARG(dbenv);
+#endif
+ if (sv) {
+ if (db_errpfx)
+ sv_setpvf(sv, "%s: %s", db_errpfx, buffer) ;
+ else
+ sv_setpv(sv, buffer) ;
+ }
+}
+#endif
+
+#if defined(TRACE) && defined(BERKELEY_DB_1_OR_2)
+
+static void
+#ifdef CAN_PROTOTYPE
+PrintHash(INFO *hash)
+#else
+PrintHash(hash)
+INFO * hash ;
+#endif
+{
+ printf ("HASH Info\n") ;
+ printf (" hash = %s\n",
+ (hash->db_HA_hash != NULL ? "redefined" : "default")) ;
+ printf (" bsize = %d\n", hash->db_HA_bsize) ;
+ printf (" ffactor = %d\n", hash->db_HA_ffactor) ;
+ printf (" nelem = %d\n", hash->db_HA_nelem) ;
+ printf (" cachesize = %d\n", hash->db_HA_cachesize) ;
+ printf (" lorder = %d\n", hash->db_HA_lorder) ;
+
+}
+
+static void
+#ifdef CAN_PROTOTYPE
+PrintRecno(INFO *recno)
+#else
+PrintRecno(recno)
+INFO * recno ;
+#endif
+{
+ printf ("RECNO Info\n") ;
+ printf (" flags = %d\n", recno->db_RE_flags) ;
+ printf (" cachesize = %d\n", recno->db_RE_cachesize) ;
+ printf (" psize = %d\n", recno->db_RE_psize) ;
+ printf (" lorder = %d\n", recno->db_RE_lorder) ;
+ printf (" reclen = %lu\n", (unsigned long)recno->db_RE_reclen) ;
+ printf (" bval = %d 0x%x\n", recno->db_RE_bval, recno->db_RE_bval) ;
+ printf (" bfname = %d [%s]\n", recno->db_RE_bfname, recno->db_RE_bfname) ;
+}
+
+static void
+#ifdef CAN_PROTOTYPE
+PrintBtree(INFO *btree)
+#else
+PrintBtree(btree)
+INFO * btree ;
+#endif
+{
+ printf ("BTREE Info\n") ;
+ printf (" compare = %s\n",
+ (btree->db_BT_compare ? "redefined" : "default")) ;
+ printf (" prefix = %s\n",
+ (btree->db_BT_prefix ? "redefined" : "default")) ;
+ printf (" flags = %d\n", btree->db_BT_flags) ;
+ printf (" cachesize = %d\n", btree->db_BT_cachesize) ;
+ printf (" psize = %d\n", btree->db_BT_psize) ;
+#ifndef DB_VERSION_MAJOR
+ printf (" maxkeypage = %d\n", btree->db_BT_maxkeypage) ;
+ printf (" minkeypage = %d\n", btree->db_BT_minkeypage) ;
+#endif
+ printf (" lorder = %d\n", btree->db_BT_lorder) ;
+}
+
+#else
+
+#define PrintRecno(recno)
+#define PrintHash(hash)
+#define PrintBtree(btree)
+
+#endif /* TRACE */
+
+
+static I32
+#ifdef CAN_PROTOTYPE
+GetArrayLength(pTHX_ DB_File db)
+#else
+GetArrayLength(db)
+DB_File db ;
+#endif
+{
+ DBT key ;
+ DBT value ;
+ int RETVAL ;
+
+ DBT_clear(key) ;
+ DBT_clear(value) ;
+ RETVAL = do_SEQ(db, key, value, R_LAST) ;
+ if (RETVAL == 0)
+ RETVAL = *(I32 *)key.data ;
+ else /* No key means empty file */
+ RETVAL = 0 ;
+
+ return ((I32)RETVAL) ;
+}
+
+static recno_t
+#ifdef CAN_PROTOTYPE
+GetRecnoKey(pTHX_ DB_File db, I32 value)
+#else
+GetRecnoKey(db, value)
+DB_File db ;
+I32 value ;
+#endif
+{
+ if (value < 0) {
+ /* Get the length of the array */
+ I32 length = GetArrayLength(aTHX_ db) ;
+
+ /* check for attempt to write before start of array */
+ if (length + value + 1 <= 0) {
+ tidyUp(db);
+ croak("Modification of non-creatable array value attempted, subscript %ld", (long)value) ;
+ }
+
+ value = length + value + 1 ;
+ }
+ else
+ ++ value ;
+
+ return value ;
+}
+
+
+static DB_File
+#ifdef CAN_PROTOTYPE
+ParseOpenInfo(pTHX_ int isHASH, char *name, int flags, int mode, SV *sv)
+#else
+ParseOpenInfo(isHASH, name, flags, mode, sv)
+int isHASH ;
+char * name ;
+int flags ;
+int mode ;
+SV * sv ;
+#endif
+{
+
+#ifdef BERKELEY_DB_1_OR_2 /* Berkeley DB Version 1 or 2 */
+
+ SV ** svp;
+ HV * action ;
+ DB_File RETVAL = (DB_File)safemalloc(sizeof(DB_File_type)) ;
+ void * openinfo = NULL ;
+ INFO * info = &RETVAL->info ;
+ STRLEN n_a;
+ dMY_CXT;
+
+#ifdef TRACE
+ printf("In ParseOpenInfo name=[%s] flags=[%d] mode=[%d] SV NULL=[%d]\n",
+ name, flags, mode, sv == NULL) ;
+#endif
+ Zero(RETVAL, 1, DB_File_type) ;
+
+ /* Default to HASH */
+ RETVAL->filtering = 0 ;
+ RETVAL->filter_fetch_key = RETVAL->filter_store_key =
+ RETVAL->filter_fetch_value = RETVAL->filter_store_value =
+ RETVAL->hash = RETVAL->compare = RETVAL->prefix = NULL ;
+ RETVAL->type = DB_HASH ;
+
+ /* DGH - Next line added to avoid SEGV on existing hash DB */
+ CurrentDB = RETVAL;
+
+ /* fd for 1.86 hash in memory files doesn't return -1 like 1.85 */
+ RETVAL->in_memory = (name == NULL) ;
+
+ if (sv)
+ {
+ if (! SvROK(sv) )
+ croak ("type parameter is not a reference") ;
+
+ svp = hv_fetch( (HV*)SvRV(sv), "GOT", 3, FALSE) ;
+ if (svp && SvOK(*svp))
+ action = (HV*) SvRV(*svp) ;
+ else
+ croak("internal error") ;
+
+ if (sv_isa(sv, "DB_File::HASHINFO"))
+ {
+
+ if (!isHASH)
+ croak("DB_File can only tie an associative array to a DB_HASH database") ;
+
+ RETVAL->type = DB_HASH ;
+ openinfo = (void*)info ;
+
+ svp = hv_fetch(action, "hash", 4, FALSE);
+
+ if (svp && SvOK(*svp))
+ {
+ info->db_HA_hash = hash_cb ;
+ RETVAL->hash = newSVsv(*svp) ;
+ }
+ else
+ info->db_HA_hash = NULL ;
+
+ svp = hv_fetch(action, "ffactor", 7, FALSE);
+ info->db_HA_ffactor = svp ? SvIV(*svp) : 0;
+
+ svp = hv_fetch(action, "nelem", 5, FALSE);
+ info->db_HA_nelem = svp ? SvIV(*svp) : 0;
+
+ svp = hv_fetch(action, "bsize", 5, FALSE);
+ info->db_HA_bsize = svp ? SvIV(*svp) : 0;
+
+ svp = hv_fetch(action, "cachesize", 9, FALSE);
+ info->db_HA_cachesize = svp ? SvIV(*svp) : 0;
+
+ svp = hv_fetch(action, "lorder", 6, FALSE);
+ info->db_HA_lorder = svp ? SvIV(*svp) : 0;
+
+ PrintHash(info) ;
+ }
+ else if (sv_isa(sv, "DB_File::BTREEINFO"))
+ {
+ if (!isHASH)
+ croak("DB_File can only tie an associative array to a DB_BTREE database");
+
+ RETVAL->type = DB_BTREE ;
+ openinfo = (void*)info ;
+
+ svp = hv_fetch(action, "compare", 7, FALSE);
+ if (svp && SvOK(*svp))
+ {
+ info->db_BT_compare = btree_compare ;
+ RETVAL->compare = newSVsv(*svp) ;
+ }
+ else
+ info->db_BT_compare = NULL ;
+
+ svp = hv_fetch(action, "prefix", 6, FALSE);
+ if (svp && SvOK(*svp))
+ {
+ info->db_BT_prefix = btree_prefix ;
+ RETVAL->prefix = newSVsv(*svp) ;
+ }
+ else
+ info->db_BT_prefix = NULL ;
+
+ svp = hv_fetch(action, "flags", 5, FALSE);
+ info->db_BT_flags = svp ? SvIV(*svp) : 0;
+
+ svp = hv_fetch(action, "cachesize", 9, FALSE);
+ info->db_BT_cachesize = svp ? SvIV(*svp) : 0;
+
+#ifndef DB_VERSION_MAJOR
+ svp = hv_fetch(action, "minkeypage", 10, FALSE);
+ info->btree.minkeypage = svp ? SvIV(*svp) : 0;
+
+ svp = hv_fetch(action, "maxkeypage", 10, FALSE);
+ info->btree.maxkeypage = svp ? SvIV(*svp) : 0;
+#endif
+
+ svp = hv_fetch(action, "psize", 5, FALSE);
+ info->db_BT_psize = svp ? SvIV(*svp) : 0;
+
+ svp = hv_fetch(action, "lorder", 6, FALSE);
+ info->db_BT_lorder = svp ? SvIV(*svp) : 0;
+
+ PrintBtree(info) ;
+
+ }
+ else if (sv_isa(sv, "DB_File::RECNOINFO"))
+ {
+ if (isHASH)
+ croak("DB_File can only tie an array to a DB_RECNO database");
+
+ RETVAL->type = DB_RECNO ;
+ openinfo = (void *)info ;
+
+ info->db_RE_flags = 0 ;
+
+ svp = hv_fetch(action, "flags", 5, FALSE);
+ info->db_RE_flags = (u_long) (svp ? SvIV(*svp) : 0);
+
+ svp = hv_fetch(action, "reclen", 6, FALSE);
+ info->db_RE_reclen = (size_t) (svp ? SvIV(*svp) : 0);
+
+ svp = hv_fetch(action, "cachesize", 9, FALSE);
+ info->db_RE_cachesize = (u_int) (svp ? SvIV(*svp) : 0);
+
+ svp = hv_fetch(action, "psize", 5, FALSE);
+ info->db_RE_psize = (u_int) (svp ? SvIV(*svp) : 0);
+
+ svp = hv_fetch(action, "lorder", 6, FALSE);
+ info->db_RE_lorder = (int) (svp ? SvIV(*svp) : 0);
+
+#ifdef DB_VERSION_MAJOR
+ info->re_source = name ;
+ name = NULL ;
+#endif
+ svp = hv_fetch(action, "bfname", 6, FALSE);
+ if (svp && SvOK(*svp)) {
+ char * ptr = SvPV(*svp,n_a) ;
+#ifdef DB_VERSION_MAJOR
+ name = (char*) n_a ? ptr : NULL ;
+#else
+ info->db_RE_bfname = (char*) (n_a ? ptr : NULL) ;
+#endif
+ }
+ else
+#ifdef DB_VERSION_MAJOR
+ name = NULL ;
+#else
+ info->db_RE_bfname = NULL ;
+#endif
+
+ svp = hv_fetch(action, "bval", 4, FALSE);
+#ifdef DB_VERSION_MAJOR
+ if (svp && SvOK(*svp))
+ {
+ int value ;
+ if (SvPOK(*svp))
+ value = (int)*SvPV(*svp, n_a) ;
+ else
+ value = SvIV(*svp) ;
+
+ if (info->flags & DB_FIXEDLEN) {
+ info->re_pad = value ;
+ info->flags |= DB_PAD ;
+ }
+ else {
+ info->re_delim = value ;
+ info->flags |= DB_DELIMITER ;
+ }
+
+ }
+#else
+ if (svp && SvOK(*svp))
+ {
+ if (SvPOK(*svp))
+ info->db_RE_bval = (u_char)*SvPV(*svp, n_a) ;
+ else
+ info->db_RE_bval = (u_char)(unsigned long) SvIV(*svp) ;
+ DB_flags(info->flags, DB_DELIMITER) ;
+
+ }
+ else
+ {
+ if (info->db_RE_flags & R_FIXEDLEN)
+ info->db_RE_bval = (u_char) ' ' ;
+ else
+ info->db_RE_bval = (u_char) '\n' ;
+ DB_flags(info->flags, DB_DELIMITER) ;
+ }
+#endif
+
+#ifdef DB_RENUMBER
+ info->flags |= DB_RENUMBER ;
+#endif
+
+ PrintRecno(info) ;
+ }
+ else
+ croak("type is not of type DB_File::HASHINFO, DB_File::BTREEINFO or DB_File::RECNOINFO");
+ }
+
+
+ /* OS2 Specific Code */
+#ifdef OS2
+#ifdef __EMX__
+ flags |= O_BINARY;
+#endif /* __EMX__ */
+#endif /* OS2 */
+
+#ifdef DB_VERSION_MAJOR
+
+ {
+ int Flags = 0 ;
+ int status ;
+
+ /* Map 1.x flags to 2.x flags */
+ if ((flags & O_CREAT) == O_CREAT)
+ Flags |= DB_CREATE ;
+
+#if O_RDONLY == 0
+ if (flags == O_RDONLY)
+#else
+ if ((flags & O_RDONLY) == O_RDONLY && (flags & O_RDWR) != O_RDWR)
+#endif
+ Flags |= DB_RDONLY ;
+
+#ifdef O_TRUNC
+ if ((flags & O_TRUNC) == O_TRUNC)
+ Flags |= DB_TRUNCATE ;
+#endif
+
+ status = db_open(name, RETVAL->type, Flags, mode, NULL, (DB_INFO*)openinfo, &RETVAL->dbp) ;
+ if (status == 0)
+#if DB_VERSION_MAJOR == 2 && DB_VERSION_MINOR < 6
+ status = (RETVAL->dbp->cursor)(RETVAL->dbp, NULL, &RETVAL->cursor) ;
+#else
+ status = (RETVAL->dbp->cursor)(RETVAL->dbp, NULL, &RETVAL->cursor,
+ 0) ;
+#endif
+
+ if (status)
+ RETVAL->dbp = NULL ;
+
+ }
+#else
+
+#if defined(DB_LIBRARY_COMPATIBILITY_API) && DB_VERSION_MAJOR > 2
+ RETVAL->dbp = __db185_open(name, flags, mode, RETVAL->type, openinfo) ;
+#else
+ RETVAL->dbp = dbopen(name, flags, mode, RETVAL->type, openinfo) ;
+#endif /* DB_LIBRARY_COMPATIBILITY_API */
+
+#endif
+
+ return (RETVAL) ;
+
+#else /* Berkeley DB Version > 2 */
+
+ SV ** svp;
+ HV * action ;
+ DB_File RETVAL = (DB_File)safemalloc(sizeof(DB_File_type)) ;
+ DB * dbp ;
+ STRLEN n_a;
+ int status ;
+ dMY_CXT;
+
+/* printf("In ParseOpenInfo name=[%s] flags=[%d] mode = [%d]\n", name, flags, mode) ; */
+ Zero(RETVAL, 1, DB_File_type) ;
+
+ /* Default to HASH */
+ RETVAL->filtering = 0 ;
+ RETVAL->filter_fetch_key = RETVAL->filter_store_key =
+ RETVAL->filter_fetch_value = RETVAL->filter_store_value =
+ RETVAL->hash = RETVAL->compare = RETVAL->prefix = NULL ;
+ RETVAL->type = DB_HASH ;
+
+ /* DGH - Next line added to avoid SEGV on existing hash DB */
+ CurrentDB = RETVAL;
+
+ /* fd for 1.86 hash in memory files doesn't return -1 like 1.85 */
+ RETVAL->in_memory = (name == NULL) ;
+
+ status = db_create(&RETVAL->dbp, NULL,0) ;
+ /* printf("db_create returned %d %s\n", status, db_strerror(status)) ; */
+ if (status) {
+ RETVAL->dbp = NULL ;
+ return (RETVAL) ;
+ }
+ dbp = RETVAL->dbp ;
+
+#ifdef WANT_ERROR
+ RETVAL->dbp->set_errcall(RETVAL->dbp, db_errcall_cb) ;
+#endif
+ if (sv)
+ {
+ if (! SvROK(sv) )
+ croak ("type parameter is not a reference") ;
+
+ svp = hv_fetch( (HV*)SvRV(sv), "GOT", 3, FALSE) ;
+ if (svp && SvOK(*svp))
+ action = (HV*) SvRV(*svp) ;
+ else
+ croak("internal error") ;
+
+ if (sv_isa(sv, "DB_File::HASHINFO"))
+ {
+
+ if (!isHASH)
+ croak("DB_File can only tie an associative array to a DB_HASH database") ;
+
+ RETVAL->type = DB_HASH ;
+
+ svp = hv_fetch(action, "hash", 4, FALSE);
+
+ if (svp && SvOK(*svp))
+ {
+ (void)dbp->set_h_hash(dbp, hash_cb) ;
+ RETVAL->hash = newSVsv(*svp) ;
+ }
+
+ svp = hv_fetch(action, "ffactor", 7, FALSE);
+ if (svp)
+ (void)dbp->set_h_ffactor(dbp, my_SvUV32(*svp)) ;
+
+ svp = hv_fetch(action, "nelem", 5, FALSE);
+ if (svp)
+ (void)dbp->set_h_nelem(dbp, my_SvUV32(*svp)) ;
+
+ svp = hv_fetch(action, "bsize", 5, FALSE);
+ if (svp)
+ (void)dbp->set_pagesize(dbp, my_SvUV32(*svp));
+
+ svp = hv_fetch(action, "cachesize", 9, FALSE);
+ if (svp)
+ (void)dbp->set_cachesize(dbp, 0, my_SvUV32(*svp), 0) ;
+
+ svp = hv_fetch(action, "lorder", 6, FALSE);
+ if (svp)
+ (void)dbp->set_lorder(dbp, (int)SvIV(*svp)) ;
+
+ PrintHash(info) ;
+ }
+ else if (sv_isa(sv, "DB_File::BTREEINFO"))
+ {
+ if (!isHASH)
+ croak("DB_File can only tie an associative array to a DB_BTREE database");
+
+ RETVAL->type = DB_BTREE ;
+
+ svp = hv_fetch(action, "compare", 7, FALSE);
+ if (svp && SvOK(*svp))
+ {
+ (void)dbp->set_bt_compare(dbp, btree_compare) ;
+ RETVAL->compare = newSVsv(*svp) ;
+ }
+
+ svp = hv_fetch(action, "prefix", 6, FALSE);
+ if (svp && SvOK(*svp))
+ {
+ (void)dbp->set_bt_prefix(dbp, btree_prefix) ;
+ RETVAL->prefix = newSVsv(*svp) ;
+ }
+
+ svp = hv_fetch(action, "flags", 5, FALSE);
+ if (svp)
+ (void)dbp->set_flags(dbp, my_SvUV32(*svp)) ;
+
+ svp = hv_fetch(action, "cachesize", 9, FALSE);
+ if (svp)
+ (void)dbp->set_cachesize(dbp, 0, my_SvUV32(*svp), 0) ;
+
+ svp = hv_fetch(action, "psize", 5, FALSE);
+ if (svp)
+ (void)dbp->set_pagesize(dbp, my_SvUV32(*svp)) ;
+
+ svp = hv_fetch(action, "lorder", 6, FALSE);
+ if (svp)
+ (void)dbp->set_lorder(dbp, (int)SvIV(*svp)) ;
+
+ PrintBtree(info) ;
+
+ }
+ else if (sv_isa(sv, "DB_File::RECNOINFO"))
+ {
+ int fixed = FALSE ;
+
+ if (isHASH)
+ croak("DB_File can only tie an array to a DB_RECNO database");
+
+ RETVAL->type = DB_RECNO ;
+
+ svp = hv_fetch(action, "flags", 5, FALSE);
+ if (svp) {
+ int flags = SvIV(*svp) ;
+ /* remove FIXDLEN, if present */
+ if (flags & DB_FIXEDLEN) {
+ fixed = TRUE ;
+ flags &= ~DB_FIXEDLEN ;
+ }
+ }
+
+ svp = hv_fetch(action, "cachesize", 9, FALSE);
+ if (svp) {
+ status = dbp->set_cachesize(dbp, 0, my_SvUV32(*svp), 0) ;
+ }
+
+ svp = hv_fetch(action, "psize", 5, FALSE);
+ if (svp) {
+ status = dbp->set_pagesize(dbp, my_SvUV32(*svp)) ;
+ }
+
+ svp = hv_fetch(action, "lorder", 6, FALSE);
+ if (svp) {
+ status = dbp->set_lorder(dbp, (int)SvIV(*svp)) ;
+ }
+
+ svp = hv_fetch(action, "bval", 4, FALSE);
+ if (svp && SvOK(*svp))
+ {
+ int value ;
+ if (SvPOK(*svp))
+ value = (int)*SvPV(*svp, n_a) ;
+ else
+ value = (int)SvIV(*svp) ;
+
+ if (fixed) {
+ status = dbp->set_re_pad(dbp, value) ;
+ }
+ else {
+ status = dbp->set_re_delim(dbp, value) ;
+ }
+
+ }
+
+ if (fixed) {
+ svp = hv_fetch(action, "reclen", 6, FALSE);
+ if (svp) {
+ u_int32_t len = my_SvUV32(*svp) ;
+ status = dbp->set_re_len(dbp, len) ;
+ }
+ }
+
+ if (name != NULL) {
+ status = dbp->set_re_source(dbp, name) ;
+ name = NULL ;
+ }
+
+ svp = hv_fetch(action, "bfname", 6, FALSE);
+ if (svp && SvOK(*svp)) {
+ char * ptr = SvPV(*svp,n_a) ;
+ name = (char*) n_a ? ptr : NULL ;
+ }
+ else
+ name = NULL ;
+
+
+ status = dbp->set_flags(dbp, (u_int32_t)DB_RENUMBER) ;
+
+ if (flags){
+ (void)dbp->set_flags(dbp, (u_int32_t)flags) ;
+ }
+ PrintRecno(info) ;
+ }
+ else
+ croak("type is not of type DB_File::HASHINFO, DB_File::BTREEINFO or DB_File::RECNOINFO");
+ }
+
+ {
+ u_int32_t Flags = 0 ;
+ int status ;
+
+ /* Map 1.x flags to 3.x flags */
+ if ((flags & O_CREAT) == O_CREAT)
+ Flags |= DB_CREATE ;
+
+#if O_RDONLY == 0
+ if (flags == O_RDONLY)
+#else
+ if ((flags & O_RDONLY) == O_RDONLY && (flags & O_RDWR) != O_RDWR)
+#endif
+ Flags |= DB_RDONLY ;
+
+#ifdef O_TRUNC
+ if ((flags & O_TRUNC) == O_TRUNC)
+ Flags |= DB_TRUNCATE ;
+#endif
+
+#ifdef AT_LEAST_DB_4_4
+ /* need this for recno */
+ if ((flags & O_TRUNC) == O_TRUNC)
+ Flags |= DB_CREATE ;
+#endif
+
+#ifdef AT_LEAST_DB_4_1
+ status = (RETVAL->dbp->open)(RETVAL->dbp, NULL, name, NULL, RETVAL->type,
+ Flags, mode) ;
+#else
+ status = (RETVAL->dbp->open)(RETVAL->dbp, name, NULL, RETVAL->type,
+ Flags, mode) ;
+#endif
+ /* printf("open returned %d %s\n", status, db_strerror(status)) ; */
+
+ if (status == 0) {
+
+ status = (RETVAL->dbp->cursor)(RETVAL->dbp, NULL, &RETVAL->cursor,
+ 0) ;
+ /* printf("cursor returned %d %s\n", status, db_strerror(status)) ; */
+ }
+
+ if (status)
+ {
+ db_close(RETVAL); /* close **dbp handle to prevent mem.leak */
+ RETVAL->dbp = NULL ;
+ }
+
+ }
+
+ return (RETVAL) ;
+
+#endif /* Berkeley DB Version > 2 */
+
+} /* ParseOpenInfo */
+
+
+#include "constants.h"
+
+#line 1550 "DB_File.c"
+#ifndef PERL_UNUSED_VAR
+# define PERL_UNUSED_VAR(var) if (0) var = var
+#endif
+
+#ifndef dVAR
+# define dVAR dNOOP
+#endif
+
+
+/* This stuff is not part of the API! You have been warned. */
+#ifndef PERL_VERSION_DECIMAL
+# define PERL_VERSION_DECIMAL(r,v,s) (r*1000000 + v*1000 + s)
+#endif
+#ifndef PERL_DECIMAL_VERSION
+# define PERL_DECIMAL_VERSION \
+ PERL_VERSION_DECIMAL(PERL_REVISION,PERL_VERSION,PERL_SUBVERSION)
+#endif
+#ifndef PERL_VERSION_GE
+# define PERL_VERSION_GE(r,v,s) \
+ (PERL_DECIMAL_VERSION >= PERL_VERSION_DECIMAL(r,v,s))
+#endif
+#ifndef PERL_VERSION_LE
+# define PERL_VERSION_LE(r,v,s) \
+ (PERL_DECIMAL_VERSION <= PERL_VERSION_DECIMAL(r,v,s))
+#endif
+
+/* XS_INTERNAL is the explicit static-linkage variant of the default
+ * XS macro.
+ *
+ * XS_EXTERNAL is the same as XS_INTERNAL except it does not include
+ * "STATIC", ie. it exports XSUB symbols. You probably don't want that
+ * for anything but the BOOT XSUB.
+ *
+ * See XSUB.h in core!
+ */
+
+
+/* TODO: This might be compatible further back than 5.10.0. */
+#if PERL_VERSION_GE(5, 10, 0) && PERL_VERSION_LE(5, 15, 1)
+# undef XS_EXTERNAL
+# undef XS_INTERNAL
+# if defined(__CYGWIN__) && defined(USE_DYNAMIC_LOADING)
+# define XS_EXTERNAL(name) __declspec(dllexport) XSPROTO(name)
+# define XS_INTERNAL(name) STATIC XSPROTO(name)
+# endif
+# if defined(__SYMBIAN32__)
+# define XS_EXTERNAL(name) EXPORT_C XSPROTO(name)
+# define XS_INTERNAL(name) EXPORT_C STATIC XSPROTO(name)
+# endif
+# ifndef XS_EXTERNAL
+# if defined(HASATTRIBUTE_UNUSED) && !defined(__cplusplus)
+# define XS_EXTERNAL(name) void name(pTHX_ CV* cv __attribute__unused__)
+# define XS_INTERNAL(name) STATIC void name(pTHX_ CV* cv __attribute__unused__)
+# else
+# ifdef __cplusplus
+# define XS_EXTERNAL(name) extern "C" XSPROTO(name)
+# define XS_INTERNAL(name) static XSPROTO(name)
+# else
+# define XS_EXTERNAL(name) XSPROTO(name)
+# define XS_INTERNAL(name) STATIC XSPROTO(name)
+# endif
+# endif
+# endif
+#endif
+
+/* perl >= 5.10.0 && perl <= 5.15.1 */
+
+
+/* The XS_EXTERNAL macro is used for functions that must not be static
+ * like the boot XSUB of a module. If perl didn't have an XS_EXTERNAL
+ * macro defined, the best we can do is assume XS is the same.
+ * Dito for XS_INTERNAL.
+ */
+#ifndef XS_EXTERNAL
+# define XS_EXTERNAL(name) XS(name)
+#endif
+#ifndef XS_INTERNAL
+# define XS_INTERNAL(name) XS(name)
+#endif
+
+/* Now, finally, after all this mess, we want an ExtUtils::ParseXS
+ * internal macro that we're free to redefine for varying linkage due
+ * to the EXPORT_XSUB_SYMBOLS XS keyword. This is internal, use
+ * XS_EXTERNAL(name) or XS_INTERNAL(name) in your code if you need to!
+ */
+
+#undef XS_EUPXS
+#if defined(PERL_EUPXS_ALWAYS_EXPORT)
+# define XS_EUPXS(name) XS_EXTERNAL(name)
+#else
+ /* default to internal */
+# define XS_EUPXS(name) XS_INTERNAL(name)
+#endif
+
+#ifndef PERL_ARGS_ASSERT_CROAK_XS_USAGE
+#define PERL_ARGS_ASSERT_CROAK_XS_USAGE assert(cv); assert(params)
+
+/* prototype to pass -Wmissing-prototypes */
+STATIC void
+S_croak_xs_usage(const CV *const cv, const char *const params);
+
+STATIC void
+S_croak_xs_usage(const CV *const cv, const char *const params)
+{
+ const GV *const gv = CvGV(cv);
+
+ PERL_ARGS_ASSERT_CROAK_XS_USAGE;
+
+ if (gv) {
+ const char *const gvname = GvNAME(gv);
+ const HV *const stash = GvSTASH(gv);
+ const char *const hvname = stash ? HvNAME(stash) : NULL;
+
+ if (hvname)
+ Perl_croak_nocontext("Usage: %s::%s(%s)", hvname, gvname, params);
+ else
+ Perl_croak_nocontext("Usage: %s(%s)", gvname, params);
+ } else {
+ /* Pants. I don't think that it should be possible to get here. */
+ Perl_croak_nocontext("Usage: CODE(0x%"UVxf")(%s)", PTR2UV(cv), params);
+ }
+}
+#undef PERL_ARGS_ASSERT_CROAK_XS_USAGE
+
+#define croak_xs_usage S_croak_xs_usage
+
+#endif
+
+/* NOTE: the prototype of newXSproto() is different in versions of perls,
+ * so we define a portable version of newXSproto()
+ */
+#ifdef newXS_flags
+#define newXSproto_portable(name, c_impl, file, proto) newXS_flags(name, c_impl, file, proto, 0)
+#else
+#define newXSproto_portable(name, c_impl, file, proto) (PL_Sv=(SV*)newXS(name, c_impl, file), sv_setpv(PL_Sv, proto), (CV*)PL_Sv)
+#endif /* !defined(newXS_flags) */
+
+#if PERL_VERSION_LE(5, 21, 5)
+# define newXS_deffile(a,b) Perl_newXS(aTHX_ a,b,file)
+#else
+# define newXS_deffile(a,b) Perl_newXS_deffile(aTHX_ a,b)
+#endif
+
+#line 1694 "DB_File.c"
+
+/* INCLUDE: Including 'constants.xs' from 'DB_File.xs' */
+
+
+XS_EUPXS(XS_DB_File_constant); /* prototype to pass -Wmissing-prototypes */
+XS_EUPXS(XS_DB_File_constant)
+{
+ dVAR; dXSARGS;
+ if (items != 1)
+ croak_xs_usage(cv, "sv");
+ PERL_UNUSED_VAR(ax); /* -Wall */
+ SP -= items;
+ {
+#line 4 "./constants.xs"
+#ifdef dXSTARG
+ dXSTARG; /* Faster if we have it. */
+#else
+ dTARGET;
+#endif
+ STRLEN len;
+ int type;
+ IV iv;
+ /* NV nv; Uncomment this if you need to return NVs */
+ /* const char *pv; Uncomment this if you need to return PVs */
+#line 1719 "DB_File.c"
+ SV * sv = ST(0)
+;
+ const char * s = SvPV(sv, len);
+#line 18 "./constants.xs"
+ /* Change this to constant(aTHX_ s, len, &iv, &nv);
+ if you need to return both NVs and IVs */
+ type = constant(aTHX_ s, len, &iv);
+ /* Return 1 or 2 items. First is error message, or undef if no error.
+ Second, if present, is found value */
+ switch (type) {
+ case PERL_constant_NOTFOUND:
+ sv =
+ sv_2mortal(newSVpvf("%s is not a valid DB_File macro", s));
+ PUSHs(sv);
+ break;
+ case PERL_constant_NOTDEF:
+ sv = sv_2mortal(newSVpvf(
+ "Your vendor has not defined DB_File macro %s, used",
+ s));
+ PUSHs(sv);
+ break;
+ case PERL_constant_ISIV:
+ EXTEND(SP, 1);
+ PUSHs(&PL_sv_undef);
+ PUSHi(iv);
+ break;
+ /* Uncomment this if you need to return NOs
+ case PERL_constant_ISNO:
+ EXTEND(SP, 1);
+ PUSHs(&PL_sv_undef);
+ PUSHs(&PL_sv_no);
+ break; */
+ /* Uncomment this if you need to return NVs
+ case PERL_constant_ISNV:
+ EXTEND(SP, 1);
+ PUSHs(&PL_sv_undef);
+ PUSHn(nv);
+ break; */
+ /* Uncomment this if you need to return PVs
+ case PERL_constant_ISPV:
+ EXTEND(SP, 1);
+ PUSHs(&PL_sv_undef);
+ PUSHp(pv, strlen(pv));
+ break; */
+ /* Uncomment this if you need to return PVNs
+ case PERL_constant_ISPVN:
+ EXTEND(SP, 1);
+ PUSHs(&PL_sv_undef);
+ PUSHp(pv, iv);
+ break; */
+ /* Uncomment this if you need to return SVs
+ case PERL_constant_ISSV:
+ EXTEND(SP, 1);
+ PUSHs(&PL_sv_undef);
+ PUSHs(sv);
+ break; */
+ /* Uncomment this if you need to return UNDEFs
+ case PERL_constant_ISUNDEF:
+ break; */
+ /* Uncomment this if you need to return UVs
+ case PERL_constant_ISUV:
+ EXTEND(SP, 1);
+ PUSHs(&PL_sv_undef);
+ PUSHu((UV)iv);
+ break; */
+ /* Uncomment this if you need to return YESs
+ case PERL_constant_ISYES:
+ EXTEND(SP, 1);
+ PUSHs(&PL_sv_undef);
+ PUSHs(&PL_sv_yes);
+ break; */
+ default:
+ sv = sv_2mortal(newSVpvf(
+ "Unexpected return type %d while processing DB_File macro %s, used",
+ type, s));
+ PUSHs(sv);
+ }
+#line 1797 "DB_File.c"
+ PUTBACK;
+ return;
+ }
+}
+
+
+/* INCLUDE: Returning to 'DB_File.xs' from 'constants.xs' */
+
+
+XS_EUPXS(XS_DB_File_DoTie_); /* prototype to pass -Wmissing-prototypes */
+XS_EUPXS(XS_DB_File_DoTie_)
+{
+ dVAR; dXSARGS;
+ if (items < 2 || items > 6)
+ croak_xs_usage(cv, "isHASH, dbtype, name=undef, flags=O_CREAT|O_RDWR, mode=0666, type=DB_HASH");
+ {
+ int isHASH = (int)SvIV(ST(0))
+;
+ char * dbtype = (char *)SvPV_nolen(ST(1))
+;
+ int flags;
+ int mode;
+ DB_File RETVAL;
+
+ if (items < 4)
+ flags = O_CREAT|O_RDWR;
+ else {
+ flags = (int)SvIV(ST(3))
+;
+ }
+
+ if (items < 5)
+ mode = 0666;
+ else {
+ mode = (int)SvIV(ST(4))
+;
+ }
+#line 1572 "DB_File.xs"
+ {
+ char * name = (char *) NULL ;
+ SV * sv = (SV *) NULL ;
+ STRLEN n_a;
+
+ if (items >= 3 && SvOK(ST(2)))
+ name = (char*) SvPV(ST(2), n_a) ;
+
+ if (items == 6)
+ sv = ST(5) ;
+
+ RETVAL = ParseOpenInfo(aTHX_ isHASH, name, flags, mode, sv) ;
+ Trace(("db_DoTie_ %p\n", RETVAL));
+ if (RETVAL->dbp == NULL) {
+ Safefree(RETVAL);
+ RETVAL = NULL ;
+ }
+ }
+#line 1854 "DB_File.c"
+ {
+ SV * RETVALSV;
+ RETVALSV = sv_newmortal();
+ sv_setref_pv(RETVALSV, dbtype, (void*)RETVAL);
+ ST(0) = RETVALSV;
+ }
+ }
+ XSRETURN(1);
+}
+
+
+XS_EUPXS(XS_DB_File_DESTROY); /* prototype to pass -Wmissing-prototypes */
+XS_EUPXS(XS_DB_File_DESTROY)
+{
+ dVAR; dXSARGS;
+ if (items != 1)
+ croak_xs_usage(cv, "db");
+ {
+ DB_File db;
+#line 1597 "DB_File.xs"
+ dMY_CXT;
+#line 1876 "DB_File.c"
+ int RETVAL;
+ dXSTARG;
+
+ if (SvROK(ST(0))) {
+ IV tmp = SvIV((SV*)SvRV(ST(0)));
+ db = INT2PTR(DB_File,tmp);
+ }
+ else
+ Perl_croak_nocontext("%s: %s is not a reference",
+ "DB_File::DESTROY",
+ "db")
+;
+#line 1599 "DB_File.xs"
+ CurrentDB = db ;
+ Trace(("DESTROY %p\n", db));
+#line 1892 "DB_File.c"
+
+ RETVAL = db_DESTROY(db);
+ XSprePUSH; PUSHi((IV)RETVAL);
+#line 1602 "DB_File.xs"
+ Trace(("DESTROY %p done\n", db));
+ if (db->hash)
+ SvREFCNT_dec(db->hash) ;
+ if (db->compare)
+ SvREFCNT_dec(db->compare) ;
+ if (db->prefix)
+ SvREFCNT_dec(db->prefix) ;
+ if (db->filter_fetch_key)
+ SvREFCNT_dec(db->filter_fetch_key) ;
+ if (db->filter_store_key)
+ SvREFCNT_dec(db->filter_store_key) ;
+ if (db->filter_fetch_value)
+ SvREFCNT_dec(db->filter_fetch_value) ;
+ if (db->filter_store_value)
+ SvREFCNT_dec(db->filter_store_value) ;
+ safefree(db) ;
+#ifdef DB_VERSION_MAJOR
+ if (RETVAL > 0)
+ RETVAL = -1 ;
+#endif
+#line 1917 "DB_File.c"
+ }
+ XSRETURN(1);
+}
+
+
+XS_EUPXS(XS_DB_File_DELETE); /* prototype to pass -Wmissing-prototypes */
+XS_EUPXS(XS_DB_File_DELETE)
+{
+ dVAR; dXSARGS;
+ if (items < 2 || items > 3)
+ croak_xs_usage(cv, "db, key, flags=0");
+ {
+ DB_File db;
+ DBTKEY key;
+ u_int flags;
+#line 1630 "DB_File.xs"
+ dMY_CXT;
+#line 1935 "DB_File.c"
+ int RETVAL;
+ dXSTARG;
+
+ if (SvROK(ST(0)) && sv_derived_from(ST(0), "DB_File")) {
+ IV tmp = SvIV((SV*)SvRV(ST(0)));
+ db = INT2PTR(DB_File,tmp);
+ }
+ else
+ Perl_croak_nocontext("%s: %s is not of type %s",
+ "DB_File::DELETE",
+ "db", "DB_File")
+;
+
+ {
+ SV * my_sv = ST(1);
+ DBM_ckFilter(my_sv, filter_store_key, "filter_store_key");
+ DBT_clear(key) ;
+ SvGETMAGIC(my_sv) ;
+ if (db->type == DB_RECNO) {
+ if (SvOK(my_sv))
+ Value = GetRecnoKey(aTHX_ db, SvIV(my_sv)) ;
+ else
+ Value = 1 ;
+ key.data = & Value;
+ key.size = (int)sizeof(recno_t);
+ }
+ else if (SvOK(my_sv)) {
+ STRLEN len;
+ key.data = SvPVbyte(my_sv, len);
+ key.size = (int)len;
+ }
+ }
+;
+
+ if (items < 3)
+ flags = 0;
+ else {
+ flags = (unsigned int)SvUV(ST(2))
+;
+ }
+#line 1632 "DB_File.xs"
+ (void)flags;
+ CurrentDB = db ;
+#line 1979 "DB_File.c"
+
+ RETVAL = db_DELETE(db, key, flags);
+ XSprePUSH; PUSHi((IV)RETVAL);
+ }
+ XSRETURN(1);
+}
+
+
+XS_EUPXS(XS_DB_File_EXISTS); /* prototype to pass -Wmissing-prototypes */
+XS_EUPXS(XS_DB_File_EXISTS)
+{
+ dVAR; dXSARGS;
+ if (items != 2)
+ croak_xs_usage(cv, "db, key");
+ {
+ DB_File db;
+ DBTKEY key;
+#line 1641 "DB_File.xs"
+ dMY_CXT;
+#line 1999 "DB_File.c"
+ int RETVAL;
+ dXSTARG;
+
+ if (SvROK(ST(0)) && sv_derived_from(ST(0), "DB_File")) {
+ IV tmp = SvIV((SV*)SvRV(ST(0)));
+ db = INT2PTR(DB_File,tmp);
+ }
+ else
+ Perl_croak_nocontext("%s: %s is not of type %s",
+ "DB_File::EXISTS",
+ "db", "DB_File")
+;
+
+ {
+ SV * my_sv = ST(1);
+ DBM_ckFilter(my_sv, filter_store_key, "filter_store_key");
+ DBT_clear(key) ;
+ SvGETMAGIC(my_sv) ;
+ if (db->type == DB_RECNO) {
+ if (SvOK(my_sv))
+ Value = GetRecnoKey(aTHX_ db, SvIV(my_sv)) ;
+ else
+ Value = 1 ;
+ key.data = & Value;
+ key.size = (int)sizeof(recno_t);
+ }
+ else if (SvOK(my_sv)) {
+ STRLEN len;
+ key.data = SvPVbyte(my_sv, len);
+ key.size = (int)len;
+ }
+ }
+;
+#line 1643 "DB_File.xs"
+ {
+ DBT value ;
+
+ DBT_clear(value) ;
+ CurrentDB = db ;
+ RETVAL = (((db->dbp)->get)(db->dbp, TXN &key, &value, 0) == 0) ;
+ }
+#line 2041 "DB_File.c"
+ XSprePUSH; PUSHi((IV)RETVAL);
+ }
+ XSRETURN(1);
+}
+
+
+XS_EUPXS(XS_DB_File_FETCH); /* prototype to pass -Wmissing-prototypes */
+XS_EUPXS(XS_DB_File_FETCH)
+{
+ dVAR; dXSARGS;
+ if (items < 2 || items > 3)
+ croak_xs_usage(cv, "db, key, flags=0");
+ {
+ DB_File db;
+ DBTKEY key;
+ u_int flags;
+#line 1659 "DB_File.xs"
+ dMY_CXT ;
+ int RETVAL ;
+#line 2061 "DB_File.c"
+
+ if (SvROK(ST(0)) && sv_derived_from(ST(0), "DB_File")) {
+ IV tmp = SvIV((SV*)SvRV(ST(0)));
+ db = INT2PTR(DB_File,tmp);
+ }
+ else
+ Perl_croak_nocontext("%s: %s is not of type %s",
+ "DB_File::FETCH",
+ "db", "DB_File")
+;
+
+ {
+ SV * my_sv = ST(1);
+ DBM_ckFilter(my_sv, filter_store_key, "filter_store_key");
+ DBT_clear(key) ;
+ SvGETMAGIC(my_sv) ;
+ if (db->type == DB_RECNO) {
+ if (SvOK(my_sv))
+ Value = GetRecnoKey(aTHX_ db, SvIV(my_sv)) ;
+ else
+ Value = 1 ;
+ key.data = & Value;
+ key.size = (int)sizeof(recno_t);
+ }
+ else if (SvOK(my_sv)) {
+ STRLEN len;
+ key.data = SvPVbyte(my_sv, len);
+ key.size = (int)len;
+ }
+ }
+;
+
+ if (items < 3)
+ flags = 0;
+ else {
+ flags = (unsigned int)SvUV(ST(2))
+;
+ }
+#line 1662 "DB_File.xs"
+ {
+ DBT value ;
+
+ DBT_clear(value) ;
+ CurrentDB = db ;
+ RETVAL = db_get(db, key, value, flags) ;
+ ST(0) = sv_newmortal();
+ OutputValue(ST(0), value)
+ }
+#line 2110 "DB_File.c"
+ }
+ XSRETURN(1);
+}
+
+
+XS_EUPXS(XS_DB_File_STORE); /* prototype to pass -Wmissing-prototypes */
+XS_EUPXS(XS_DB_File_STORE)
+{
+ dVAR; dXSARGS;
+ if (items < 3 || items > 4)
+ croak_xs_usage(cv, "db, key, value, flags=0");
+ {
+ DB_File db;
+ DBTKEY key;
+ DBT value;
+ u_int flags;
+#line 1679 "DB_File.xs"
+ dMY_CXT;
+#line 2129 "DB_File.c"
+ int RETVAL;
+ dXSTARG;
+
+ if (SvROK(ST(0)) && sv_derived_from(ST(0), "DB_File")) {
+ IV tmp = SvIV((SV*)SvRV(ST(0)));
+ db = INT2PTR(DB_File,tmp);
+ }
+ else
+ Perl_croak_nocontext("%s: %s is not of type %s",
+ "DB_File::STORE",
+ "db", "DB_File")
+;
+
+ {
+ SV * my_sv = ST(1);
+ DBM_ckFilter(my_sv, filter_store_key, "filter_store_key");
+ DBT_clear(key) ;
+ SvGETMAGIC(my_sv) ;
+ if (db->type == DB_RECNO) {
+ if (SvOK(my_sv))
+ Value = GetRecnoKey(aTHX_ db, SvIV(my_sv)) ;
+ else
+ Value = 1 ;
+ key.data = & Value;
+ key.size = (int)sizeof(recno_t);
+ }
+ else if (SvOK(my_sv)) {
+ STRLEN len;
+ key.data = SvPVbyte(my_sv, len);
+ key.size = (int)len;
+ }
+ }
+;
+
+ {
+ SV * my_sv = ST(2);
+ DBM_ckFilter(my_sv, filter_store_value, "filter_store_value");
+ DBT_clear(value) ;
+ SvGETMAGIC(my_sv) ;
+ if (SvOK(my_sv)) {
+ STRLEN len;
+ value.data = SvPVbyte(my_sv, len);
+ value.size = (int)len;
+ }
+ }
+;
+
+ if (items < 4)
+ flags = 0;
+ else {
+ flags = (unsigned int)SvUV(ST(3))
+;
+ }
+#line 1681 "DB_File.xs"
+ (void)flags;
+ CurrentDB = db ;
+#line 2186 "DB_File.c"
+
+ RETVAL = db_STORE(db, key, value, flags);
+ XSprePUSH; PUSHi((IV)RETVAL);
+ }
+ XSRETURN(1);
+}
+
+
+XS_EUPXS(XS_DB_File_FIRSTKEY); /* prototype to pass -Wmissing-prototypes */
+XS_EUPXS(XS_DB_File_FIRSTKEY)
+{
+ dVAR; dXSARGS;
+ if (items != 1)
+ croak_xs_usage(cv, "db");
+ {
+ DB_File db;
+#line 1689 "DB_File.xs"
+ dMY_CXT ;
+ int RETVAL ;
+#line 2206 "DB_File.c"
+
+ if (SvROK(ST(0)) && sv_derived_from(ST(0), "DB_File")) {
+ IV tmp = SvIV((SV*)SvRV(ST(0)));
+ db = INT2PTR(DB_File,tmp);
+ }
+ else
+ Perl_croak_nocontext("%s: %s is not of type %s",
+ "DB_File::FIRSTKEY",
+ "db", "DB_File")
+;
+#line 1692 "DB_File.xs"
+ {
+ DBTKEY key ;
+ DBT value ;
+
+ DBT_clear(key) ;
+ DBT_clear(value) ;
+ CurrentDB = db ;
+ RETVAL = do_SEQ(db, key, value, R_FIRST) ;
+ ST(0) = sv_newmortal();
+ OutputKey(ST(0), key) ;
+ }
+#line 2229 "DB_File.c"
+ }
+ XSRETURN(1);
+}
+
+
+XS_EUPXS(XS_DB_File_NEXTKEY); /* prototype to pass -Wmissing-prototypes */
+XS_EUPXS(XS_DB_File_NEXTKEY)
+{
+ dVAR; dXSARGS;
+ if (items != 2)
+ croak_xs_usage(cv, "db, key");
+ {
+ DB_File db;
+ DBTKEY key;
+#line 1709 "DB_File.xs"
+ dMY_CXT ;
+ int RETVAL ;
+#line 2247 "DB_File.c"
+
+ if (SvROK(ST(0)) && sv_derived_from(ST(0), "DB_File")) {
+ IV tmp = SvIV((SV*)SvRV(ST(0)));
+ db = INT2PTR(DB_File,tmp);
+ }
+ else
+ Perl_croak_nocontext("%s: %s is not of type %s",
+ "DB_File::NEXTKEY",
+ "db", "DB_File")
+;
+#line 1712 "DB_File.xs"
+ {
+ DBT value ;
+
+ DBT_clear(key) ;
+ DBT_clear(value) ;
+ CurrentDB = db ;
+ RETVAL = do_SEQ(db, key, value, R_NEXT) ;
+ ST(0) = sv_newmortal();
+ OutputKey(ST(0), key) ;
+ }
+#line 2269 "DB_File.c"
+ }
+ XSRETURN(1);
+}
+
+
+XS_EUPXS(XS_DB_File_unshift); /* prototype to pass -Wmissing-prototypes */
+XS_EUPXS(XS_DB_File_unshift)
+{
+ dVAR; dXSARGS;
+ dXSI32;
+ if (items < 1)
+ croak_xs_usage(cv, "db, ...");
+ {
+ DB_File db;
+#line 1732 "DB_File.xs"
+ dMY_CXT;
+#line 2286 "DB_File.c"
+ int RETVAL;
+ dXSTARG;
+
+ if (SvROK(ST(0)) && sv_derived_from(ST(0), "DB_File")) {
+ IV tmp = SvIV((SV*)SvRV(ST(0)));
+ db = INT2PTR(DB_File,tmp);
+ }
+ else
+ Perl_croak_nocontext("%s: %s is not of type %s",
+ GvNAME(CvGV(cv)),
+ "db", "DB_File")
+;
+#line 1734 "DB_File.xs"
+ {
+ DBTKEY key ;
+ DBT value ;
+ int i ;
+ int One ;
+ STRLEN n_a;
+
+ DBT_clear(key) ;
+ DBT_clear(value) ;
+ CurrentDB = db ;
+#ifdef DB_VERSION_MAJOR
+ /* get the first value */
+ RETVAL = do_SEQ(db, key, value, DB_FIRST) ;
+ RETVAL = 0 ;
+#else
+ RETVAL = -1 ;
+#endif
+ for (i = items-1 ; i > 0 ; --i)
+ {
+ DBM_ckFilter(ST(i), filter_store_value, "filter_store_value");
+ value.data = SvPVbyte(ST(i), n_a) ;
+ value.size = n_a ;
+ One = 1 ;
+ key.data = &One ;
+ key.size = sizeof(int) ;
+#ifdef DB_VERSION_MAJOR
+ RETVAL = (db->cursor->c_put)(db->cursor, &key, &value, DB_BEFORE) ;
+#else
+ RETVAL = (db->dbp->put)(db->dbp, &key, &value, R_IBEFORE) ;
+#endif
+ if (RETVAL != 0)
+ break;
+ }
+ }
+#line 2334 "DB_File.c"
+ XSprePUSH; PUSHi((IV)RETVAL);
+ }
+ XSRETURN(1);
+}
+
+
+XS_EUPXS(XS_DB_File_pop); /* prototype to pass -Wmissing-prototypes */
+XS_EUPXS(XS_DB_File_pop)
+{
+ dVAR; dXSARGS;
+ dXSI32;
+ if (items != 1)
+ croak_xs_usage(cv, "db");
+ {
+ DB_File db;
+#line 1775 "DB_File.xs"
+ dMY_CXT;
+#line 2352 "DB_File.c"
+#line 1778 "DB_File.xs"
+ I32 RETVAL;
+#line 2355 "DB_File.c"
+
+ if (SvROK(ST(0)) && sv_derived_from(ST(0), "DB_File")) {
+ IV tmp = SvIV((SV*)SvRV(ST(0)));
+ db = INT2PTR(DB_File,tmp);
+ }
+ else
+ Perl_croak_nocontext("%s: %s is not of type %s",
+ GvNAME(CvGV(cv)),
+ "db", "DB_File")
+;
+#line 1780 "DB_File.xs"
+ {
+ DBTKEY key ;
+ DBT value ;
+
+ DBT_clear(key) ;
+ DBT_clear(value) ;
+ CurrentDB = db ;
+
+ /* First get the final value */
+ RETVAL = do_SEQ(db, key, value, R_LAST) ;
+ ST(0) = sv_newmortal();
+ /* Now delete it */
+ if (RETVAL == 0)
+ {
+ /* the call to del will trash value, so take a copy now */
+ OutputValue(ST(0), value) ;
+ RETVAL = db_del(db, key, R_CURSOR) ;
+ if (RETVAL != 0)
+ sv_setsv(ST(0), &PL_sv_undef);
+ }
+ }
+#line 2388 "DB_File.c"
+ }
+ XSRETURN(1);
+}
+
+
+XS_EUPXS(XS_DB_File_shift); /* prototype to pass -Wmissing-prototypes */
+XS_EUPXS(XS_DB_File_shift)
+{
+ dVAR; dXSARGS;
+ dXSI32;
+ if (items != 1)
+ croak_xs_usage(cv, "db");
+ {
+ DB_File db;
+#line 1806 "DB_File.xs"
+ dMY_CXT;
+#line 2405 "DB_File.c"
+#line 1809 "DB_File.xs"
+ I32 RETVAL;
+#line 2408 "DB_File.c"
+
+ if (SvROK(ST(0)) && sv_derived_from(ST(0), "DB_File")) {
+ IV tmp = SvIV((SV*)SvRV(ST(0)));
+ db = INT2PTR(DB_File,tmp);
+ }
+ else
+ Perl_croak_nocontext("%s: %s is not of type %s",
+ GvNAME(CvGV(cv)),
+ "db", "DB_File")
+;
+#line 1811 "DB_File.xs"
+ {
+ DBT value ;
+ DBTKEY key ;
+
+ DBT_clear(key) ;
+ DBT_clear(value) ;
+ CurrentDB = db ;
+ /* get the first value */
+ RETVAL = do_SEQ(db, key, value, R_FIRST) ;
+ ST(0) = sv_newmortal();
+ /* Now delete it */
+ if (RETVAL == 0)
+ {
+ /* the call to del will trash value, so take a copy now */
+ OutputValue(ST(0), value) ;
+ RETVAL = db_del(db, key, R_CURSOR) ;
+ if (RETVAL != 0)
+ sv_setsv (ST(0), &PL_sv_undef) ;
+ }
+ }
+#line 2440 "DB_File.c"
+ }
+ XSRETURN(1);
+}
+
+
+XS_EUPXS(XS_DB_File_push); /* prototype to pass -Wmissing-prototypes */
+XS_EUPXS(XS_DB_File_push)
+{
+ dVAR; dXSARGS;
+ dXSI32;
+ if (items < 1)
+ croak_xs_usage(cv, "db, ...");
+ {
+ DB_File db;
+#line 1837 "DB_File.xs"
+ dMY_CXT;
+#line 2457 "DB_File.c"
+ I32 RETVAL;
+ dXSTARG;
+
+ if (SvROK(ST(0)) && sv_derived_from(ST(0), "DB_File")) {
+ IV tmp = SvIV((SV*)SvRV(ST(0)));
+ db = INT2PTR(DB_File,tmp);
+ }
+ else
+ Perl_croak_nocontext("%s: %s is not of type %s",
+ GvNAME(CvGV(cv)),
+ "db", "DB_File")
+;
+#line 1840 "DB_File.xs"
+ {
+ DBTKEY key ;
+ DBT value ;
+ DB * Db = db->dbp ;
+ int i ;
+ STRLEN n_a;
+ int keyval ;
+
+ DBT_flags(key) ;
+ DBT_flags(value) ;
+ CurrentDB = db ;
+ /* Set the Cursor to the Last element */
+ RETVAL = do_SEQ(db, key, value, R_LAST) ;
+#ifndef DB_VERSION_MAJOR
+ if (RETVAL >= 0)
+#endif
+ {
+ if (RETVAL == 0)
+ keyval = *(int*)key.data ;
+ else
+ keyval = 0 ;
+ for (i = 1 ; i < items ; ++i)
+ {
+ DBM_ckFilter(ST(i), filter_store_value, "filter_store_value");
+ value.data = SvPVbyte(ST(i), n_a) ;
+ value.size = n_a ;
+ ++ keyval ;
+ key.data = &keyval ;
+ key.size = sizeof(int) ;
+ RETVAL = (Db->put)(Db, TXN &key, &value, 0) ;
+ if (RETVAL != 0)
+ break;
+ }
+ }
+ }
+#line 2506 "DB_File.c"
+ XSprePUSH; PUSHi((IV)RETVAL);
+ }
+ XSRETURN(1);
+}
+
+
+XS_EUPXS(XS_DB_File_length); /* prototype to pass -Wmissing-prototypes */
+XS_EUPXS(XS_DB_File_length)
+{
+ dVAR; dXSARGS;
+ dXSI32;
+ if (items != 1)
+ croak_xs_usage(cv, "db");
+ {
+ DB_File db;
+#line 1882 "DB_File.xs"
+ dMY_CXT;
+#line 2524 "DB_File.c"
+ I32 RETVAL;
+ dXSTARG;
+
+ if (SvROK(ST(0)) && sv_derived_from(ST(0), "DB_File")) {
+ IV tmp = SvIV((SV*)SvRV(ST(0)));
+ db = INT2PTR(DB_File,tmp);
+ }
+ else
+ Perl_croak_nocontext("%s: %s is not of type %s",
+ GvNAME(CvGV(cv)),
+ "db", "DB_File")
+;
+#line 1885 "DB_File.xs"
+ CurrentDB = db ;
+ RETVAL = GetArrayLength(aTHX_ db) ;
+#line 2540 "DB_File.c"
+ XSprePUSH; PUSHi((IV)RETVAL);
+ }
+ XSRETURN(1);
+}
+
+
+XS_EUPXS(XS_DB_File_del); /* prototype to pass -Wmissing-prototypes */
+XS_EUPXS(XS_DB_File_del)
+{
+ dVAR; dXSARGS;
+ if (items < 2 || items > 3)
+ croak_xs_usage(cv, "db, key, flags=0");
+ {
+ DB_File db;
+ DBTKEY key;
+ u_int flags;
+#line 1901 "DB_File.xs"
+ dMY_CXT;
+#line 2559 "DB_File.c"
+ int RETVAL;
+ dXSTARG;
+
+ if (SvROK(ST(0)) && sv_derived_from(ST(0), "DB_File")) {
+ IV tmp = SvIV((SV*)SvRV(ST(0)));
+ db = INT2PTR(DB_File,tmp);
+ }
+ else
+ Perl_croak_nocontext("%s: %s is not of type %s",
+ "DB_File::del",
+ "db", "DB_File")
+;
+
+ {
+ SV * my_sv = ST(1);
+ DBM_ckFilter(my_sv, filter_store_key, "filter_store_key");
+ DBT_clear(key) ;
+ SvGETMAGIC(my_sv) ;
+ if (db->type == DB_RECNO) {
+ if (SvOK(my_sv))
+ Value = GetRecnoKey(aTHX_ db, SvIV(my_sv)) ;
+ else
+ Value = 1 ;
+ key.data = & Value;
+ key.size = (int)sizeof(recno_t);
+ }
+ else if (SvOK(my_sv)) {
+ STRLEN len;
+ key.data = SvPVbyte(my_sv, len);
+ key.size = (int)len;
+ }
+ }
+;
+
+ if (items < 3)
+ flags = 0;
+ else {
+ flags = (unsigned int)SvUV(ST(2))
+;
+ }
+#line 1903 "DB_File.xs"
+ CurrentDB = db ;
+ RETVAL = db_del(db, key, flags) ;
+#ifdef DB_VERSION_MAJOR
+ if (RETVAL > 0)
+ RETVAL = -1 ;
+ else if (RETVAL == DB_NOTFOUND)
+ RETVAL = 1 ;
+#endif
+#line 2609 "DB_File.c"
+ XSprePUSH; PUSHi((IV)RETVAL);
+ }
+ XSRETURN(1);
+}
+
+
+XS_EUPXS(XS_DB_File_get); /* prototype to pass -Wmissing-prototypes */
+XS_EUPXS(XS_DB_File_get)
+{
+ dVAR; dXSARGS;
+ if (items < 3 || items > 4)
+ croak_xs_usage(cv, "db, key, value, flags=0");
+ {
+ DB_File db;
+ DBTKEY key;
+ DBT value;
+ u_int flags;
+#line 1922 "DB_File.xs"
+ dMY_CXT;
+#line 2629 "DB_File.c"
+ int RETVAL;
+ dXSTARG;
+
+ if (SvROK(ST(0)) && sv_derived_from(ST(0), "DB_File")) {
+ IV tmp = SvIV((SV*)SvRV(ST(0)));
+ db = INT2PTR(DB_File,tmp);
+ }
+ else
+ Perl_croak_nocontext("%s: %s is not of type %s",
+ "DB_File::get",
+ "db", "DB_File")
+;
+
+ {
+ SV * my_sv = ST(1);
+ DBM_ckFilter(my_sv, filter_store_key, "filter_store_key");
+ DBT_clear(key) ;
+ SvGETMAGIC(my_sv) ;
+ if (db->type == DB_RECNO) {
+ if (SvOK(my_sv))
+ Value = GetRecnoKey(aTHX_ db, SvIV(my_sv)) ;
+ else
+ Value = 1 ;
+ key.data = & Value;
+ key.size = (int)sizeof(recno_t);
+ }
+ else if (SvOK(my_sv)) {
+ STRLEN len;
+ key.data = SvPVbyte(my_sv, len);
+ key.size = (int)len;
+ }
+ }
+;
+
+ if (items < 4)
+ flags = 0;
+ else {
+ flags = (unsigned int)SvUV(ST(3))
+;
+ }
+#line 1924 "DB_File.xs"
+ CurrentDB = db ;
+ DBT_clear(value) ;
+ RETVAL = db_get(db, key, value, flags) ;
+#ifdef DB_VERSION_MAJOR
+ if (RETVAL > 0)
+ RETVAL = -1 ;
+ else if (RETVAL == DB_NOTFOUND)
+ RETVAL = 1 ;
+#endif
+#line 2680 "DB_File.c"
+ OutputValue(ST(2), value)
+ SvSETMAGIC(ST(2));
+ XSprePUSH; PUSHi((IV)RETVAL);
+ }
+ XSRETURN(1);
+}
+
+
+XS_EUPXS(XS_DB_File_put); /* prototype to pass -Wmissing-prototypes */
+XS_EUPXS(XS_DB_File_put)
+{
+ dVAR; dXSARGS;
+ if (items < 3 || items > 4)
+ croak_xs_usage(cv, "db, key, value, flags=0");
+ {
+ DB_File db;
+ DBTKEY key;
+ DBT value;
+ u_int flags;
+#line 1944 "DB_File.xs"
+ dMY_CXT;
+#line 2702 "DB_File.c"
+ int RETVAL;
+ dXSTARG;
+
+ if (SvROK(ST(0)) && sv_derived_from(ST(0), "DB_File")) {
+ IV tmp = SvIV((SV*)SvRV(ST(0)));
+ db = INT2PTR(DB_File,tmp);
+ }
+ else
+ Perl_croak_nocontext("%s: %s is not of type %s",
+ "DB_File::put",
+ "db", "DB_File")
+;
+
+ {
+ SV * my_sv = ST(1);
+ DBM_ckFilter(my_sv, filter_store_key, "filter_store_key");
+ DBT_clear(key) ;
+ SvGETMAGIC(my_sv) ;
+ if (db->type == DB_RECNO) {
+ if (SvOK(my_sv))
+ Value = GetRecnoKey(aTHX_ db, SvIV(my_sv)) ;
+ else
+ Value = 1 ;
+ key.data = & Value;
+ key.size = (int)sizeof(recno_t);
+ }
+ else if (SvOK(my_sv)) {
+ STRLEN len;
+ key.data = SvPVbyte(my_sv, len);
+ key.size = (int)len;
+ }
+ }
+;
+
+ {
+ SV * my_sv = ST(2);
+ DBM_ckFilter(my_sv, filter_store_value, "filter_store_value");
+ DBT_clear(value) ;
+ SvGETMAGIC(my_sv) ;
+ if (SvOK(my_sv)) {
+ STRLEN len;
+ value.data = SvPVbyte(my_sv, len);
+ value.size = (int)len;
+ }
+ }
+;
+
+ if (items < 4)
+ flags = 0;
+ else {
+ flags = (unsigned int)SvUV(ST(3))
+;
+ }
+#line 1946 "DB_File.xs"
+ CurrentDB = db ;
+ RETVAL = db_put(db, key, value, flags) ;
+#ifdef DB_VERSION_MAJOR
+ if (RETVAL > 0)
+ RETVAL = -1 ;
+ else if (RETVAL == DB_KEYEXIST)
+ RETVAL = 1 ;
+#endif
+#line 2765 "DB_File.c"
+ if (flagSet(flags, R_IAFTER) || flagSet(flags, R_IBEFORE)) OutputKey(ST(1), key);
+ SvSETMAGIC(ST(1));
+ XSprePUSH; PUSHi((IV)RETVAL);
+ }
+ XSRETURN(1);
+}
+
+
+XS_EUPXS(XS_DB_File_fd); /* prototype to pass -Wmissing-prototypes */
+XS_EUPXS(XS_DB_File_fd)
+{
+ dVAR; dXSARGS;
+ if (items != 1)
+ croak_xs_usage(cv, "db");
+ {
+ DB_File db;
+#line 1962 "DB_File.xs"
+ dMY_CXT ;
+#line 2784 "DB_File.c"
+ int RETVAL;
+ dXSTARG;
+
+ if (SvROK(ST(0)) && sv_derived_from(ST(0), "DB_File")) {
+ IV tmp = SvIV((SV*)SvRV(ST(0)));
+ db = INT2PTR(DB_File,tmp);
+ }
+ else
+ Perl_croak_nocontext("%s: %s is not of type %s",
+ "DB_File::fd",
+ "db", "DB_File")
+;
+#line 1964 "DB_File.xs"
+ CurrentDB = db ;
+#ifdef DB_VERSION_MAJOR
+ RETVAL = -1 ;
+ {
+ int status = 0 ;
+ status = (db->in_memory
+ ? -1
+ : ((db->dbp)->fd)(db->dbp, &RETVAL) ) ;
+ if (status != 0)
+ RETVAL = -1 ;
+ }
+#else
+ RETVAL = (db->in_memory
+ ? -1
+ : ((db->dbp)->fd)(db->dbp) ) ;
+#endif
+#line 2814 "DB_File.c"
+ XSprePUSH; PUSHi((IV)RETVAL);
+ }
+ XSRETURN(1);
+}
+
+
+XS_EUPXS(XS_DB_File_sync); /* prototype to pass -Wmissing-prototypes */
+XS_EUPXS(XS_DB_File_sync)
+{
+ dVAR; dXSARGS;
+ if (items < 1 || items > 2)
+ croak_xs_usage(cv, "db, flags=0");
+ {
+ DB_File db;
+ u_int flags;
+#line 1988 "DB_File.xs"
+ dMY_CXT;
+#line 2832 "DB_File.c"
+ int RETVAL;
+ dXSTARG;
+
+ if (SvROK(ST(0)) && sv_derived_from(ST(0), "DB_File")) {
+ IV tmp = SvIV((SV*)SvRV(ST(0)));
+ db = INT2PTR(DB_File,tmp);
+ }
+ else
+ Perl_croak_nocontext("%s: %s is not of type %s",
+ "DB_File::sync",
+ "db", "DB_File")
+;
+
+ if (items < 2)
+ flags = 0;
+ else {
+ flags = (unsigned int)SvUV(ST(1))
+;
+ }
+#line 1990 "DB_File.xs"
+ CurrentDB = db ;
+ RETVAL = db_sync(db, flags) ;
+#ifdef DB_VERSION_MAJOR
+ if (RETVAL > 0)
+ RETVAL = -1 ;
+#endif
+#line 2859 "DB_File.c"
+ XSprePUSH; PUSHi((IV)RETVAL);
+ }
+ XSRETURN(1);
+}
+
+
+XS_EUPXS(XS_DB_File_seq); /* prototype to pass -Wmissing-prototypes */
+XS_EUPXS(XS_DB_File_seq)
+{
+ dVAR; dXSARGS;
+ if (items != 4)
+ croak_xs_usage(cv, "db, key, value, flags");
+ {
+ DB_File db;
+ DBTKEY key;
+ DBT value;
+ u_int flags = (unsigned int)SvUV(ST(3))
+;
+#line 2007 "DB_File.xs"
+ dMY_CXT;
+#line 2880 "DB_File.c"
+ int RETVAL;
+ dXSTARG;
+
+ if (SvROK(ST(0)) && sv_derived_from(ST(0), "DB_File")) {
+ IV tmp = SvIV((SV*)SvRV(ST(0)));
+ db = INT2PTR(DB_File,tmp);
+ }
+ else
+ Perl_croak_nocontext("%s: %s is not of type %s",
+ "DB_File::seq",
+ "db", "DB_File")
+;
+
+ {
+ SV * my_sv = ST(1);
+ DBM_ckFilter(my_sv, filter_store_key, "filter_store_key");
+ DBT_clear(key) ;
+ SvGETMAGIC(my_sv) ;
+ if (db->type == DB_RECNO) {
+ if (SvOK(my_sv))
+ Value = GetRecnoKey(aTHX_ db, SvIV(my_sv)) ;
+ else
+ Value = 1 ;
+ key.data = & Value;
+ key.size = (int)sizeof(recno_t);
+ }
+ else if (SvOK(my_sv)) {
+ STRLEN len;
+ key.data = SvPVbyte(my_sv, len);
+ key.size = (int)len;
+ }
+ }
+;
+#line 2009 "DB_File.xs"
+ CurrentDB = db ;
+ DBT_clear(value) ;
+ RETVAL = db_seq(db, key, value, flags);
+#ifdef DB_VERSION_MAJOR
+ if (RETVAL > 0)
+ RETVAL = -1 ;
+ else if (RETVAL == DB_NOTFOUND)
+ RETVAL = 1 ;
+#endif
+#line 2924 "DB_File.c"
+ OutputKey(ST(1), key)
+ SvSETMAGIC(ST(1));
+ OutputValue(ST(2), value)
+ SvSETMAGIC(ST(2));
+ XSprePUSH; PUSHi((IV)RETVAL);
+ }
+ XSRETURN(1);
+}
+
+
+XS_EUPXS(XS_DB_File_filter_fetch_key); /* prototype to pass -Wmissing-prototypes */
+XS_EUPXS(XS_DB_File_filter_fetch_key)
+{
+ dVAR; dXSARGS;
+ if (items != 2)
+ croak_xs_usage(cv, "db, code");
+ {
+ DB_File db;
+ SV * code = ST(1)
+;
+ SV * RETVAL = &PL_sv_undef ;
+
+ if (SvROK(ST(0)) && sv_derived_from(ST(0), "DB_File")) {
+ IV tmp = SvIV((SV*)SvRV(ST(0)));
+ db = INT2PTR(DB_File,tmp);
+ }
+ else
+ Perl_croak_nocontext("%s: %s is not of type %s",
+ "DB_File::filter_fetch_key",
+ "db", "DB_File")
+;
+#line 2029 "DB_File.xs"
+ DBM_setFilter(db->filter_fetch_key, code) ;
+#line 2958 "DB_File.c"
+ }
+ XSRETURN(1);
+}
+
+
+XS_EUPXS(XS_DB_File_filter_store_key); /* prototype to pass -Wmissing-prototypes */
+XS_EUPXS(XS_DB_File_filter_store_key)
+{
+ dVAR; dXSARGS;
+ if (items != 2)
+ croak_xs_usage(cv, "db, code");
+ {
+ DB_File db;
+ SV * code = ST(1)
+;
+ SV * RETVAL = &PL_sv_undef ;
+
+ if (SvROK(ST(0)) && sv_derived_from(ST(0), "DB_File")) {
+ IV tmp = SvIV((SV*)SvRV(ST(0)));
+ db = INT2PTR(DB_File,tmp);
+ }
+ else
+ Perl_croak_nocontext("%s: %s is not of type %s",
+ "DB_File::filter_store_key",
+ "db", "DB_File")
+;
+#line 2037 "DB_File.xs"
+ DBM_setFilter(db->filter_store_key, code) ;
+#line 2987 "DB_File.c"
+ }
+ XSRETURN(1);
+}
+
+
+XS_EUPXS(XS_DB_File_filter_fetch_value); /* prototype to pass -Wmissing-prototypes */
+XS_EUPXS(XS_DB_File_filter_fetch_value)
+{
+ dVAR; dXSARGS;
+ if (items != 2)
+ croak_xs_usage(cv, "db, code");
+ {
+ DB_File db;
+ SV * code = ST(1)
+;
+ SV * RETVAL = &PL_sv_undef ;
+
+ if (SvROK(ST(0)) && sv_derived_from(ST(0), "DB_File")) {
+ IV tmp = SvIV((SV*)SvRV(ST(0)));
+ db = INT2PTR(DB_File,tmp);
+ }
+ else
+ Perl_croak_nocontext("%s: %s is not of type %s",
+ "DB_File::filter_fetch_value",
+ "db", "DB_File")
+;
+#line 2045 "DB_File.xs"
+ DBM_setFilter(db->filter_fetch_value, code) ;
+#line 3016 "DB_File.c"
+ }
+ XSRETURN(1);
+}
+
+
+XS_EUPXS(XS_DB_File_filter_store_value); /* prototype to pass -Wmissing-prototypes */
+XS_EUPXS(XS_DB_File_filter_store_value)
+{
+ dVAR; dXSARGS;
+ if (items != 2)
+ croak_xs_usage(cv, "db, code");
+ {
+ DB_File db;
+ SV * code = ST(1)
+;
+ SV * RETVAL = &PL_sv_undef ;
+
+ if (SvROK(ST(0)) && sv_derived_from(ST(0), "DB_File")) {
+ IV tmp = SvIV((SV*)SvRV(ST(0)));
+ db = INT2PTR(DB_File,tmp);
+ }
+ else
+ Perl_croak_nocontext("%s: %s is not of type %s",
+ "DB_File::filter_store_value",
+ "db", "DB_File")
+;
+#line 2053 "DB_File.xs"
+ DBM_setFilter(db->filter_store_value, code) ;
+#line 3045 "DB_File.c"
+ }
+ XSRETURN(1);
+}
+
+#ifdef __cplusplus
+extern "C"
+#endif
+XS_EXTERNAL(boot_DB_File); /* prototype to pass -Wmissing-prototypes */
+XS_EXTERNAL(boot_DB_File)
+{
+#if PERL_VERSION_LE(5, 21, 5)
+ dVAR; dXSARGS;
+#else
+ dVAR; dXSBOOTARGSXSAPIVERCHK;
+#endif
+#if (PERL_REVISION == 5 && PERL_VERSION < 9)
+ char* file = __FILE__;
+#else
+ const char* file = __FILE__;
+#endif
+
+ PERL_UNUSED_VAR(file);
+
+ PERL_UNUSED_VAR(cv); /* -W */
+ PERL_UNUSED_VAR(items); /* -W */
+#if PERL_VERSION_LE(5, 21, 5)
+ XS_VERSION_BOOTCHECK;
+# ifdef XS_APIVERSION_BOOTCHECK
+ XS_APIVERSION_BOOTCHECK;
+# endif
+#endif
+
+ newXS_deffile("DB_File::constant", XS_DB_File_constant);
+ newXS_deffile("DB_File::DoTie_", XS_DB_File_DoTie_);
+ newXS_deffile("DB_File::DESTROY", XS_DB_File_DESTROY);
+ newXS_deffile("DB_File::DELETE", XS_DB_File_DELETE);
+ newXS_deffile("DB_File::EXISTS", XS_DB_File_EXISTS);
+ newXS_deffile("DB_File::FETCH", XS_DB_File_FETCH);
+ newXS_deffile("DB_File::STORE", XS_DB_File_STORE);
+ newXS_deffile("DB_File::FIRSTKEY", XS_DB_File_FIRSTKEY);
+ newXS_deffile("DB_File::NEXTKEY", XS_DB_File_NEXTKEY);
+ cv = newXS_deffile("DB_File::UNSHIFT", XS_DB_File_unshift);
+ XSANY.any_i32 = 1;
+ cv = newXS_deffile("DB_File::unshift", XS_DB_File_unshift);
+ XSANY.any_i32 = 0;
+ cv = newXS_deffile("DB_File::POP", XS_DB_File_pop);
+ XSANY.any_i32 = 1;
+ cv = newXS_deffile("DB_File::pop", XS_DB_File_pop);
+ XSANY.any_i32 = 0;
+ cv = newXS_deffile("DB_File::SHIFT", XS_DB_File_shift);
+ XSANY.any_i32 = 1;
+ cv = newXS_deffile("DB_File::shift", XS_DB_File_shift);
+ XSANY.any_i32 = 0;
+ cv = newXS_deffile("DB_File::PUSH", XS_DB_File_push);
+ XSANY.any_i32 = 1;
+ cv = newXS_deffile("DB_File::push", XS_DB_File_push);
+ XSANY.any_i32 = 0;
+ cv = newXS_deffile("DB_File::FETCHSIZE", XS_DB_File_length);
+ XSANY.any_i32 = 1;
+ cv = newXS_deffile("DB_File::length", XS_DB_File_length);
+ XSANY.any_i32 = 0;
+ newXS_deffile("DB_File::del", XS_DB_File_del);
+ newXS_deffile("DB_File::get", XS_DB_File_get);
+ newXS_deffile("DB_File::put", XS_DB_File_put);
+ newXS_deffile("DB_File::fd", XS_DB_File_fd);
+ newXS_deffile("DB_File::sync", XS_DB_File_sync);
+ newXS_deffile("DB_File::seq", XS_DB_File_seq);
+ newXS_deffile("DB_File::filter_fetch_key", XS_DB_File_filter_fetch_key);
+ newXS_deffile("DB_File::filter_store_key", XS_DB_File_filter_store_key);
+ newXS_deffile("DB_File::filter_fetch_value", XS_DB_File_filter_fetch_value);
+ newXS_deffile("DB_File::filter_store_value", XS_DB_File_filter_store_value);
+
+ /* Initialisation Section */
+
+#line 1545 "DB_File.xs"
+ {
+#ifdef dTHX
+ dTHX;
+#endif
+#ifdef WANT_ERROR
+ SV * sv_err = perl_get_sv(ERR_BUFF, GV_ADD|GV_ADDMULTI) ;
+#endif
+ MY_CXT_INIT;
+#ifdef WANT_ERROR
+ PERL_UNUSED_VAR(sv_err); /* huh? we just retrieved it... */
+#endif
+ __getBerkeleyDBInfo() ;
+
+ DBT_clear(empty) ;
+ empty.data = &zero ;
+ empty.size = sizeof(recno_t) ;
+ }
+
+#line 3139 "DB_File.c"
+
+ /* End of Initialisation Section */
+
+#if PERL_VERSION_LE(5, 21, 5)
+# if PERL_VERSION_GE(5, 9, 0)
+ if (PL_unitcheckav)
+ call_list(PL_scopestack_ix, PL_unitcheckav);
+# endif
+ XSRETURN_YES;
+#else
+ Perl_xs_boot_epilog(aTHX_ ax);
+#endif
+}
+
diff --git a/fastSum/resources/ROUGE/DB_File-1.835/DB_File.o b/fastSum/resources/ROUGE/DB_File-1.835/DB_File.o
new file mode 100644
index 0000000000000000000000000000000000000000..2e309dab415e7e47c2774e2dc91840e89c4db906
Binary files /dev/null and b/fastSum/resources/ROUGE/DB_File-1.835/DB_File.o differ
diff --git a/fastSum/resources/ROUGE/DB_File-1.835/DB_File.pm b/fastSum/resources/ROUGE/DB_File-1.835/DB_File.pm
new file mode 100644
index 0000000000000000000000000000000000000000..9b1f9577386e0d81bd798ad6f9023fe512f36839
--- /dev/null
+++ b/fastSum/resources/ROUGE/DB_File-1.835/DB_File.pm
@@ -0,0 +1,2310 @@
+# DB_File.pm -- Perl 5 interface to Berkeley DB
+#
+# Written by Paul Marquess (pmqs@cpan.org)
+#
+# Copyright (c) 1995-2014 Paul Marquess. All rights reserved.
+# This program is free software; you can redistribute it and/or
+# modify it under the same terms as Perl itself.
+
+
+package DB_File::HASHINFO ;
+
+require 5.008003;
+
+use warnings;
+use strict;
+use Carp;
+require Tie::Hash;
+@DB_File::HASHINFO::ISA = qw(Tie::Hash);
+
+sub new
+{
+ my $pkg = shift ;
+ my %x ;
+ tie %x, $pkg ;
+ bless \%x, $pkg ;
+}
+
+
+sub TIEHASH
+{
+ my $pkg = shift ;
+
+ bless { VALID => {
+ bsize => 1,
+ ffactor => 1,
+ nelem => 1,
+ cachesize => 1,
+ hash => 2,
+ lorder => 1,
+ },
+ GOT => {}
+ }, $pkg ;
+}
+
+
+sub FETCH
+{
+ my $self = shift ;
+ my $key = shift ;
+
+ return $self->{GOT}{$key} if exists $self->{VALID}{$key} ;
+
+ my $pkg = ref $self ;
+ croak "${pkg}::FETCH - Unknown element '$key'" ;
+}
+
+
+sub STORE
+{
+ my $self = shift ;
+ my $key = shift ;
+ my $value = shift ;
+
+ my $type = $self->{VALID}{$key};
+
+ if ( $type )
+ {
+ croak "Key '$key' not associated with a code reference"
+ if $type == 2 && !ref $value && ref $value ne 'CODE';
+ $self->{GOT}{$key} = $value ;
+ return ;
+ }
+
+ my $pkg = ref $self ;
+ croak "${pkg}::STORE - Unknown element '$key'" ;
+}
+
+sub DELETE
+{
+ my $self = shift ;
+ my $key = shift ;
+
+ if ( exists $self->{VALID}{$key} )
+ {
+ delete $self->{GOT}{$key} ;
+ return ;
+ }
+
+ my $pkg = ref $self ;
+ croak "DB_File::HASHINFO::DELETE - Unknown element '$key'" ;
+}
+
+sub EXISTS
+{
+ my $self = shift ;
+ my $key = shift ;
+
+ exists $self->{VALID}{$key} ;
+}
+
+sub NotHere
+{
+ my $self = shift ;
+ my $method = shift ;
+
+ croak ref($self) . " does not define the method ${method}" ;
+}
+
+sub FIRSTKEY { my $self = shift ; $self->NotHere("FIRSTKEY") }
+sub NEXTKEY { my $self = shift ; $self->NotHere("NEXTKEY") }
+sub CLEAR { my $self = shift ; $self->NotHere("CLEAR") }
+
+package DB_File::RECNOINFO ;
+
+use warnings;
+use strict ;
+
+@DB_File::RECNOINFO::ISA = qw(DB_File::HASHINFO) ;
+
+sub TIEHASH
+{
+ my $pkg = shift ;
+
+ bless { VALID => { map {$_, 1}
+ qw( bval cachesize psize flags lorder reclen bfname )
+ },
+ GOT => {},
+ }, $pkg ;
+}
+
+package DB_File::BTREEINFO ;
+
+use warnings;
+use strict ;
+
+@DB_File::BTREEINFO::ISA = qw(DB_File::HASHINFO) ;
+
+sub TIEHASH
+{
+ my $pkg = shift ;
+
+ bless { VALID => {
+ flags => 1,
+ cachesize => 1,
+ maxkeypage => 1,
+ minkeypage => 1,
+ psize => 1,
+ compare => 2,
+ prefix => 2,
+ lorder => 1,
+ },
+ GOT => {},
+ }, $pkg ;
+}
+
+
+package DB_File ;
+
+use warnings;
+use strict;
+our ($VERSION, @ISA, @EXPORT, $AUTOLOAD, $DB_BTREE, $DB_HASH, $DB_RECNO);
+our ($db_version, $use_XSLoader, $splice_end_array_no_length, $splice_end_array, $Error);
+use Carp;
+
+
+$VERSION = "1.835" ;
+$VERSION = eval $VERSION; # needed for dev releases
+
+{
+ local $SIG{__WARN__} = sub {$splice_end_array_no_length = join(" ",@_);};
+ my @a =(1); splice(@a, 3);
+ $splice_end_array_no_length =
+ ($splice_end_array_no_length =~ /^splice\(\) offset past end of array at /);
+}
+{
+ local $SIG{__WARN__} = sub {$splice_end_array = join(" ", @_);};
+ my @a =(1); splice(@a, 3, 1);
+ $splice_end_array =
+ ($splice_end_array =~ /^splice\(\) offset past end of array at /);
+}
+
+#typedef enum { DB_BTREE, DB_HASH, DB_RECNO } DBTYPE;
+$DB_BTREE = new DB_File::BTREEINFO ;
+$DB_HASH = new DB_File::HASHINFO ;
+$DB_RECNO = new DB_File::RECNOINFO ;
+
+require Tie::Hash;
+require Exporter;
+BEGIN {
+ $use_XSLoader = 1 ;
+ { local $SIG{__DIE__} ; eval { require XSLoader } ; }
+
+ if ($@) {
+ $use_XSLoader = 0 ;
+ require DynaLoader;
+ @ISA = qw(DynaLoader);
+ }
+}
+
+push @ISA, qw(Tie::Hash Exporter);
+@EXPORT = qw(
+ $DB_BTREE $DB_HASH $DB_RECNO
+
+ BTREEMAGIC
+ BTREEVERSION
+ DB_LOCK
+ DB_SHMEM
+ DB_TXN
+ HASHMAGIC
+ HASHVERSION
+ MAX_PAGE_NUMBER
+ MAX_PAGE_OFFSET
+ MAX_REC_NUMBER
+ RET_ERROR
+ RET_SPECIAL
+ RET_SUCCESS
+ R_CURSOR
+ R_DUP
+ R_FIRST
+ R_FIXEDLEN
+ R_IAFTER
+ R_IBEFORE
+ R_LAST
+ R_NEXT
+ R_NOKEY
+ R_NOOVERWRITE
+ R_PREV
+ R_RECNOSYNC
+ R_SETCURSOR
+ R_SNAPSHOT
+ __R_UNUSED
+
+);
+
+sub AUTOLOAD {
+ my($constname);
+ ($constname = $AUTOLOAD) =~ s/.*:://;
+ my ($error, $val) = constant($constname);
+ Carp::croak $error if $error;
+ no strict 'refs';
+ *{$AUTOLOAD} = sub { $val };
+ goto &{$AUTOLOAD};
+}
+
+
+eval {
+ # Make all Fcntl O_XXX constants available for importing
+ require Fcntl;
+ my @O = grep /^O_/, @Fcntl::EXPORT;
+ Fcntl->import(@O); # first we import what we want to export
+ push(@EXPORT, @O);
+};
+
+if ($use_XSLoader)
+ { XSLoader::load("DB_File", $VERSION)}
+else
+ { bootstrap DB_File $VERSION }
+
+sub tie_hash_or_array
+{
+ my (@arg) = @_ ;
+ my $tieHASH = ( (caller(1))[3] =~ /TIEHASH/ ) ;
+
+ use File::Spec;
+ $arg[1] = File::Spec->rel2abs($arg[1])
+ if defined $arg[1] ;
+
+ $arg[4] = tied %{ $arg[4] }
+ if @arg >= 5 && ref $arg[4] && $arg[4] =~ /=HASH/ && tied %{ $arg[4] } ;
+
+ $arg[2] = O_CREAT()|O_RDWR() if @arg >=3 && ! defined $arg[2];
+ $arg[3] = 0666 if @arg >=4 && ! defined $arg[3];
+
+ # make recno in Berkeley DB version 2 (or better) work like
+ # recno in version 1.
+ if ($db_version >= 4 and ! $tieHASH) {
+ $arg[2] |= O_CREAT();
+ }
+
+ if ($db_version > 1 and defined $arg[4] and $arg[4] =~ /RECNO/ and
+ $arg[1] and ! -e $arg[1]) {
+ open(FH, ">$arg[1]") or return undef ;
+ close FH ;
+ chmod $arg[3] ? $arg[3] : 0666 , $arg[1] ;
+ }
+
+ DoTie_($tieHASH, @arg) ;
+}
+
+sub TIEHASH
+{
+ tie_hash_or_array(@_) ;
+}
+
+sub TIEARRAY
+{
+ tie_hash_or_array(@_) ;
+}
+
+sub CLEAR
+{
+ my $self = shift;
+ my $key = 0 ;
+ my $value = "" ;
+ my $status = $self->seq($key, $value, R_FIRST());
+ my @keys;
+
+ while ($status == 0) {
+ push @keys, $key;
+ $status = $self->seq($key, $value, R_NEXT());
+ }
+ foreach $key (reverse @keys) {
+ my $s = $self->del($key);
+ }
+}
+
+sub EXTEND { }
+
+sub STORESIZE
+{
+ my $self = shift;
+ my $length = shift ;
+ my $current_length = $self->length() ;
+
+ if ($length < $current_length) {
+ my $key ;
+ for ($key = $current_length - 1 ; $key >= $length ; -- $key)
+ { $self->del($key) }
+ }
+ elsif ($length > $current_length) {
+ $self->put($length-1, "") ;
+ }
+}
+
+
+sub SPLICE
+{
+ my $self = shift;
+ my $offset = shift;
+ if (not defined $offset) {
+ warnings::warnif('uninitialized', 'Use of uninitialized value in splice');
+ $offset = 0;
+ }
+
+ my $has_length = @_;
+ my $length = @_ ? shift : 0;
+ # Carping about definedness comes _after_ the OFFSET sanity check.
+ # This is so we get the same error messages as Perl's splice().
+ #
+
+ my @list = @_;
+
+ my $size = $self->FETCHSIZE();
+
+ # 'If OFFSET is negative then it start that far from the end of
+ # the array.'
+ #
+ if ($offset < 0) {
+ my $new_offset = $size + $offset;
+ if ($new_offset < 0) {
+ die "Modification of non-creatable array value attempted, "
+ . "subscript $offset";
+ }
+ $offset = $new_offset;
+ }
+
+ if (not defined $length) {
+ warnings::warnif('uninitialized', 'Use of uninitialized value in splice');
+ $length = 0;
+ }
+
+ if ($offset > $size) {
+ $offset = $size;
+ warnings::warnif('misc', 'splice() offset past end of array')
+ if $has_length ? $splice_end_array : $splice_end_array_no_length;
+ }
+
+ # 'If LENGTH is omitted, removes everything from OFFSET onward.'
+ if (not defined $length) {
+ $length = $size - $offset;
+ }
+
+ # 'If LENGTH is negative, leave that many elements off the end of
+ # the array.'
+ #
+ if ($length < 0) {
+ $length = $size - $offset + $length;
+
+ if ($length < 0) {
+ # The user must have specified a length bigger than the
+ # length of the array passed in. But perl's splice()
+ # doesn't catch this, it just behaves as for length=0.
+ #
+ $length = 0;
+ }
+ }
+
+ if ($length > $size - $offset) {
+ $length = $size - $offset;
+ }
+
+ # $num_elems holds the current number of elements in the database.
+ my $num_elems = $size;
+
+ # 'Removes the elements designated by OFFSET and LENGTH from an
+ # array,'...
+ #
+ my @removed = ();
+ foreach (0 .. $length - 1) {
+ my $old;
+ my $status = $self->get($offset, $old);
+ if ($status != 0) {
+ my $msg = "error from Berkeley DB on get($offset, \$old)";
+ if ($status == 1) {
+ $msg .= ' (no such element?)';
+ }
+ else {
+ $msg .= ": error status $status";
+ if (defined $! and $! ne '') {
+ $msg .= ", message $!";
+ }
+ }
+ die $msg;
+ }
+ push @removed, $old;
+
+ $status = $self->del($offset);
+ if ($status != 0) {
+ my $msg = "error from Berkeley DB on del($offset)";
+ if ($status == 1) {
+ $msg .= ' (no such element?)';
+ }
+ else {
+ $msg .= ": error status $status";
+ if (defined $! and $! ne '') {
+ $msg .= ", message $!";
+ }
+ }
+ die $msg;
+ }
+
+ -- $num_elems;
+ }
+
+ # ...'and replaces them with the elements of LIST, if any.'
+ my $pos = $offset;
+ while (defined (my $elem = shift @list)) {
+ my $old_pos = $pos;
+ my $status;
+ if ($pos >= $num_elems) {
+ $status = $self->put($pos, $elem);
+ }
+ else {
+ $status = $self->put($pos, $elem, $self->R_IBEFORE);
+ }
+
+ if ($status != 0) {
+ my $msg = "error from Berkeley DB on put($pos, $elem, ...)";
+ if ($status == 1) {
+ $msg .= ' (no such element?)';
+ }
+ else {
+ $msg .= ", error status $status";
+ if (defined $! and $! ne '') {
+ $msg .= ", message $!";
+ }
+ }
+ die $msg;
+ }
+
+ die "pos unexpectedly changed from $old_pos to $pos with R_IBEFORE"
+ if $old_pos != $pos;
+
+ ++ $pos;
+ ++ $num_elems;
+ }
+
+ if (wantarray) {
+ # 'In list context, returns the elements removed from the
+ # array.'
+ #
+ return @removed;
+ }
+ elsif (defined wantarray and not wantarray) {
+ # 'In scalar context, returns the last element removed, or
+ # undef if no elements are removed.'
+ #
+ if (@removed) {
+ my $last = pop @removed;
+ return "$last";
+ }
+ else {
+ return undef;
+ }
+ }
+ elsif (not defined wantarray) {
+ # Void context
+ }
+ else { die }
+}
+sub ::DB_File::splice { &SPLICE }
+
+sub find_dup
+{
+ croak "Usage: \$db->find_dup(key,value)\n"
+ unless @_ == 3 ;
+
+ my $db = shift ;
+ my ($origkey, $value_wanted) = @_ ;
+ my ($key, $value) = ($origkey, 0);
+ my ($status) = 0 ;
+
+ for ($status = $db->seq($key, $value, R_CURSOR() ) ;
+ $status == 0 ;
+ $status = $db->seq($key, $value, R_NEXT() ) ) {
+
+ return 0 if $key eq $origkey and $value eq $value_wanted ;
+ }
+
+ return $status ;
+}
+
+sub del_dup
+{
+ croak "Usage: \$db->del_dup(key,value)\n"
+ unless @_ == 3 ;
+
+ my $db = shift ;
+ my ($key, $value) = @_ ;
+ my ($status) = $db->find_dup($key, $value) ;
+ return $status if $status != 0 ;
+
+ $status = $db->del($key, R_CURSOR() ) ;
+ return $status ;
+}
+
+sub get_dup
+{
+ croak "Usage: \$db->get_dup(key [,flag])\n"
+ unless @_ == 2 or @_ == 3 ;
+
+ my $db = shift ;
+ my $key = shift ;
+ my $flag = shift ;
+ my $value = 0 ;
+ my $origkey = $key ;
+ my $wantarray = wantarray ;
+ my %values = () ;
+ my @values = () ;
+ my $counter = 0 ;
+ my $status = 0 ;
+
+ # iterate through the database until either EOF ($status == 0)
+ # or a different key is encountered ($key ne $origkey).
+ for ($status = $db->seq($key, $value, R_CURSOR()) ;
+ $status == 0 and $key eq $origkey ;
+ $status = $db->seq($key, $value, R_NEXT()) ) {
+
+ # save the value or count number of matches
+ if ($wantarray) {
+ if ($flag)
+ { ++ $values{$value} }
+ else
+ { push (@values, $value) }
+ }
+ else
+ { ++ $counter }
+
+ }
+
+ return ($wantarray ? ($flag ? %values : @values) : $counter) ;
+}
+
+
+sub STORABLE_freeze
+{
+ my $type = ref shift;
+ croak "Cannot freeze $type object\n";
+}
+
+sub STORABLE_thaw
+{
+ my $type = ref shift;
+ croak "Cannot thaw $type object\n";
+}
+
+
+
+1;
+__END__
+
+=head1 NAME
+
+DB_File - Perl5 access to Berkeley DB version 1.x
+
+=head1 SYNOPSIS
+
+ use DB_File;
+
+ [$X =] tie %hash, 'DB_File', [$filename, $flags, $mode, $DB_HASH] ;
+ [$X =] tie %hash, 'DB_File', $filename, $flags, $mode, $DB_BTREE ;
+ [$X =] tie @array, 'DB_File', $filename, $flags, $mode, $DB_RECNO ;
+
+ $status = $X->del($key [, $flags]) ;
+ $status = $X->put($key, $value [, $flags]) ;
+ $status = $X->get($key, $value [, $flags]) ;
+ $status = $X->seq($key, $value, $flags) ;
+ $status = $X->sync([$flags]) ;
+ $status = $X->fd ;
+
+ # BTREE only
+ $count = $X->get_dup($key) ;
+ @list = $X->get_dup($key) ;
+ %list = $X->get_dup($key, 1) ;
+ $status = $X->find_dup($key, $value) ;
+ $status = $X->del_dup($key, $value) ;
+
+ # RECNO only
+ $a = $X->length;
+ $a = $X->pop ;
+ $X->push(list);
+ $a = $X->shift;
+ $X->unshift(list);
+ @r = $X->splice(offset, length, elements);
+
+ # DBM Filters
+ $old_filter = $db->filter_store_key ( sub { ... } ) ;
+ $old_filter = $db->filter_store_value( sub { ... } ) ;
+ $old_filter = $db->filter_fetch_key ( sub { ... } ) ;
+ $old_filter = $db->filter_fetch_value( sub { ... } ) ;
+
+ untie %hash ;
+ untie @array ;
+
+=head1 DESCRIPTION
+
+B is a module which allows Perl programs to make use of the
+facilities provided by Berkeley DB version 1.x (if you have a newer
+version of DB, see L).
+It is assumed that you have a copy of the Berkeley DB manual pages at
+hand when reading this documentation. The interface defined here
+mirrors the Berkeley DB interface closely.
+
+Berkeley DB is a C library which provides a consistent interface to a
+number of database formats. B provides an interface to all
+three of the database types currently supported by Berkeley DB.
+
+The file types are:
+
+=over 5
+
+=item B
+
+This database type allows arbitrary key/value pairs to be stored in data
+files. This is equivalent to the functionality provided by other
+hashing packages like DBM, NDBM, ODBM, GDBM, and SDBM. Remember though,
+the files created using DB_HASH are not compatible with any of the
+other packages mentioned.
+
+A default hashing algorithm, which will be adequate for most
+applications, is built into Berkeley DB. If you do need to use your own
+hashing algorithm it is possible to write your own in Perl and have
+B use it instead.
+
+=item B
+
+The btree format allows arbitrary key/value pairs to be stored in a
+sorted, balanced binary tree.
+
+As with the DB_HASH format, it is possible to provide a user defined
+Perl routine to perform the comparison of keys. By default, though, the
+keys are stored in lexical order.
+
+=item B
+
+DB_RECNO allows both fixed-length and variable-length flat text files
+to be manipulated using the same key/value pair interface as in DB_HASH
+and DB_BTREE. In this case the key will consist of a record (line)
+number.
+
+=back
+
+=head2 Using DB_File with Berkeley DB version 2 or greater
+
+Although B is intended to be used with Berkeley DB version 1,
+it can also be used with version 2, 3 or 4. In this case the interface is
+limited to the functionality provided by Berkeley DB 1.x. Anywhere the
+version 2 or greater interface differs, B arranges for it to work
+like version 1. This feature allows B scripts that were built
+with version 1 to be migrated to version 2 or greater without any changes.
+
+If you want to make use of the new features available in Berkeley DB
+2.x or greater, use the Perl module B instead.
+
+B The database file format has changed multiple times in Berkeley
+DB version 2, 3 and 4. If you cannot recreate your databases, you
+must dump any existing databases with either the C or the
+C utility that comes with Berkeley DB.
+Once you have rebuilt DB_File to use Berkeley DB version 2 or greater,
+your databases can be recreated using C. Refer to the Berkeley DB
+documentation for further details.
+
+Please read L<"COPYRIGHT"> before using version 2.x or greater of Berkeley
+DB with DB_File.
+
+=head2 Interface to Berkeley DB
+
+B allows access to Berkeley DB files using the tie() mechanism
+in Perl 5 (for full details, see L). This facility
+allows B to access Berkeley DB files using either an
+associative array (for DB_HASH & DB_BTREE file types) or an ordinary
+array (for the DB_RECNO file type).
+
+In addition to the tie() interface, it is also possible to access most
+of the functions provided in the Berkeley DB API directly.
+See L.
+
+=head2 Opening a Berkeley DB Database File
+
+Berkeley DB uses the function dbopen() to open or create a database.
+Here is the C prototype for dbopen():
+
+ DB*
+ dbopen (const char * file, int flags, int mode,
+ DBTYPE type, const void * openinfo)
+
+The parameter C is an enumeration which specifies which of the 3
+interface methods (DB_HASH, DB_BTREE or DB_RECNO) is to be used.
+Depending on which of these is actually chosen, the final parameter,
+I points to a data structure which allows tailoring of the
+specific interface method.
+
+This interface is handled slightly differently in B. Here is
+an equivalent call using B:
+
+ tie %array, 'DB_File', $filename, $flags, $mode, $DB_HASH ;
+
+The C, C and C parameters are the direct
+equivalent of their dbopen() counterparts. The final parameter $DB_HASH
+performs the function of both the C and C parameters in
+dbopen().
+
+In the example above $DB_HASH is actually a pre-defined reference to a
+hash object. B has three of these pre-defined references.
+Apart from $DB_HASH, there is also $DB_BTREE and $DB_RECNO.
+
+The keys allowed in each of these pre-defined references is limited to
+the names used in the equivalent C structure. So, for example, the
+$DB_HASH reference will only allow keys called C, C,
+C, C, C and C.
+
+To change one of these elements, just assign to it like this:
+
+ $DB_HASH->{'cachesize'} = 10000 ;
+
+The three predefined variables $DB_HASH, $DB_BTREE and $DB_RECNO are
+usually adequate for most applications. If you do need to create extra
+instances of these objects, constructors are available for each file
+type.
+
+Here are examples of the constructors and the valid options available
+for DB_HASH, DB_BTREE and DB_RECNO respectively.
+
+ $a = new DB_File::HASHINFO ;
+ $a->{'bsize'} ;
+ $a->{'cachesize'} ;
+ $a->{'ffactor'};
+ $a->{'hash'} ;
+ $a->{'lorder'} ;
+ $a->{'nelem'} ;
+
+ $b = new DB_File::BTREEINFO ;
+ $b->{'flags'} ;
+ $b->{'cachesize'} ;
+ $b->{'maxkeypage'} ;
+ $b->{'minkeypage'} ;
+ $b->{'psize'} ;
+ $b->{'compare'} ;
+ $b->{'prefix'} ;
+ $b->{'lorder'} ;
+
+ $c = new DB_File::RECNOINFO ;
+ $c->{'bval'} ;
+ $c->{'cachesize'} ;
+ $c->{'psize'} ;
+ $c->{'flags'} ;
+ $c->{'lorder'} ;
+ $c->{'reclen'} ;
+ $c->{'bfname'} ;
+
+The values stored in the hashes above are mostly the direct equivalent
+of their C counterpart. Like their C counterparts, all are set to a
+default values - that means you don't have to set I of the
+values when you only want to change one. Here is an example:
+
+ $a = new DB_File::HASHINFO ;
+ $a->{'cachesize'} = 12345 ;
+ tie %y, 'DB_File', "filename", $flags, 0777, $a ;
+
+A few of the options need extra discussion here. When used, the C
+equivalent of the keys C, C and C store pointers
+to C functions. In B these keys are used to store references
+to Perl subs. Below are templates for each of the subs:
+
+ sub hash
+ {
+ my ($data) = @_ ;
+ ...
+ # return the hash value for $data
+ return $hash ;
+ }
+
+ sub compare
+ {
+ my ($key, $key2) = @_ ;
+ ...
+ # return 0 if $key1 eq $key2
+ # -1 if $key1 lt $key2
+ # 1 if $key1 gt $key2
+ return (-1 , 0 or 1) ;
+ }
+
+ sub prefix
+ {
+ my ($key, $key2) = @_ ;
+ ...
+ # return number of bytes of $key2 which are
+ # necessary to determine that it is greater than $key1
+ return $bytes ;
+ }
+
+See L for an example of using the
+C template.
+
+If you are using the DB_RECNO interface and you intend making use of
+C, you should check out L.
+
+=head2 Default Parameters
+
+It is possible to omit some or all of the final 4 parameters in the
+call to C and let them take default values. As DB_HASH is the most
+common file format used, the call:
+
+ tie %A, "DB_File", "filename" ;
+
+is equivalent to:
+
+ tie %A, "DB_File", "filename", O_CREAT|O_RDWR, 0666, $DB_HASH ;
+
+It is also possible to omit the filename parameter as well, so the
+call:
+
+ tie %A, "DB_File" ;
+
+is equivalent to:
+
+ tie %A, "DB_File", undef, O_CREAT|O_RDWR, 0666, $DB_HASH ;
+
+See L for a discussion on the use of C
+in place of a filename.
+
+=head2 In Memory Databases
+
+Berkeley DB allows the creation of in-memory databases by using NULL
+(that is, a C<(char *)0> in C) in place of the filename. B
+uses C instead of NULL to provide this functionality.
+
+=head1 DB_HASH
+
+The DB_HASH file format is probably the most commonly used of the three
+file formats that B supports. It is also very straightforward
+to use.
+
+=head2 A Simple Example
+
+This example shows how to create a database, add key/value pairs to the
+database, delete keys/value pairs and finally how to enumerate the
+contents of the database.
+
+ use warnings ;
+ use strict ;
+ use DB_File ;
+ our (%h, $k, $v) ;
+
+ unlink "fruit" ;
+ tie %h, "DB_File", "fruit", O_RDWR|O_CREAT, 0666, $DB_HASH
+ or die "Cannot open file 'fruit': $!\n";
+
+ # Add a few key/value pairs to the file
+ $h{"apple"} = "red" ;
+ $h{"orange"} = "orange" ;
+ $h{"banana"} = "yellow" ;
+ $h{"tomato"} = "red" ;
+
+ # Check for existence of a key
+ print "Banana Exists\n\n" if $h{"banana"} ;
+
+ # Delete a key/value pair.
+ delete $h{"apple"} ;
+
+ # print the contents of the file
+ while (($k, $v) = each %h)
+ { print "$k -> $v\n" }
+
+ untie %h ;
+
+here is the output:
+
+ Banana Exists
+
+ orange -> orange
+ tomato -> red
+ banana -> yellow
+
+Note that the like ordinary associative arrays, the order of the keys
+retrieved is in an apparently random order.
+
+=head1 DB_BTREE
+
+The DB_BTREE format is useful when you want to store data in a given
+order. By default the keys will be stored in lexical order, but as you
+will see from the example shown in the next section, it is very easy to
+define your own sorting function.
+
+=head2 Changing the BTREE sort order
+
+This script shows how to override the default sorting algorithm that
+BTREE uses. Instead of using the normal lexical ordering, a case
+insensitive compare function will be used.
+
+ use warnings ;
+ use strict ;
+ use DB_File ;
+
+ my %h ;
+
+ sub Compare
+ {
+ my ($key1, $key2) = @_ ;
+ "\L$key1" cmp "\L$key2" ;
+ }
+
+ # specify the Perl sub that will do the comparison
+ $DB_BTREE->{'compare'} = \&Compare ;
+
+ unlink "tree" ;
+ tie %h, "DB_File", "tree", O_RDWR|O_CREAT, 0666, $DB_BTREE
+ or die "Cannot open file 'tree': $!\n" ;
+
+ # Add a key/value pair to the file
+ $h{'Wall'} = 'Larry' ;
+ $h{'Smith'} = 'John' ;
+ $h{'mouse'} = 'mickey' ;
+ $h{'duck'} = 'donald' ;
+
+ # Delete
+ delete $h{"duck"} ;
+
+ # Cycle through the keys printing them in order.
+ # Note it is not necessary to sort the keys as
+ # the btree will have kept them in order automatically.
+ foreach (keys %h)
+ { print "$_\n" }
+
+ untie %h ;
+
+Here is the output from the code above.
+
+ mouse
+ Smith
+ Wall
+
+There are a few point to bear in mind if you want to change the
+ordering in a BTREE database:
+
+=over 5
+
+=item 1.
+
+The new compare function must be specified when you create the database.
+
+=item 2.
+
+You cannot change the ordering once the database has been created. Thus
+you must use the same compare function every time you access the
+database.
+
+=item 3
+
+Duplicate keys are entirely defined by the comparison function.
+In the case-insensitive example above, the keys: 'KEY' and 'key'
+would be considered duplicates, and assigning to the second one
+would overwrite the first. If duplicates are allowed for (with the
+R_DUP flag discussed below), only a single copy of duplicate keys
+is stored in the database --- so (again with example above) assigning
+three values to the keys: 'KEY', 'Key', and 'key' would leave just
+the first key: 'KEY' in the database with three values. For some
+situations this results in information loss, so care should be taken
+to provide fully qualified comparison functions when necessary.
+For example, the above comparison routine could be modified to
+additionally compare case-sensitively if two keys are equal in the
+case insensitive comparison:
+
+ sub compare {
+ my($key1, $key2) = @_;
+ lc $key1 cmp lc $key2 ||
+ $key1 cmp $key2;
+ }
+
+And now you will only have duplicates when the keys themselves
+are truly the same. (note: in versions of the db library prior to
+about November 1996, such duplicate keys were retained so it was
+possible to recover the original keys in sets of keys that
+compared as equal).
+
+
+=back
+
+=head2 Handling Duplicate Keys
+
+The BTREE file type optionally allows a single key to be associated
+with an arbitrary number of values. This option is enabled by setting
+the flags element of C<$DB_BTREE> to R_DUP when creating the database.
+
+There are some difficulties in using the tied hash interface if you
+want to manipulate a BTREE database with duplicate keys. Consider this
+code:
+
+ use warnings ;
+ use strict ;
+ use DB_File ;
+
+ my ($filename, %h) ;
+
+ $filename = "tree" ;
+ unlink $filename ;
+
+ # Enable duplicate records
+ $DB_BTREE->{'flags'} = R_DUP ;
+
+ tie %h, "DB_File", $filename, O_RDWR|O_CREAT, 0666, $DB_BTREE
+ or die "Cannot open $filename: $!\n";
+
+ # Add some key/value pairs to the file
+ $h{'Wall'} = 'Larry' ;
+ $h{'Wall'} = 'Brick' ; # Note the duplicate key
+ $h{'Wall'} = 'Brick' ; # Note the duplicate key and value
+ $h{'Smith'} = 'John' ;
+ $h{'mouse'} = 'mickey' ;
+
+ # iterate through the associative array
+ # and print each key/value pair.
+ foreach (sort keys %h)
+ { print "$_ -> $h{$_}\n" }
+
+ untie %h ;
+
+Here is the output:
+
+ Smith -> John
+ Wall -> Larry
+ Wall -> Larry
+ Wall -> Larry
+ mouse -> mickey
+
+As you can see 3 records have been successfully created with key C
+- the only thing is, when they are retrieved from the database they
+I to have the same value, namely C. The problem is caused
+by the way that the associative array interface works. Basically, when
+the associative array interface is used to fetch the value associated
+with a given key, it will only ever retrieve the first value.
+
+Although it may not be immediately obvious from the code above, the
+associative array interface can be used to write values with duplicate
+keys, but it cannot be used to read them back from the database.
+
+The way to get around this problem is to use the Berkeley DB API method
+called C. This method allows sequential access to key/value
+pairs. See L for details of both the C method
+and the API in general.
+
+Here is the script above rewritten using the C API method.
+
+ use warnings ;
+ use strict ;
+ use DB_File ;
+
+ my ($filename, $x, %h, $status, $key, $value) ;
+
+ $filename = "tree" ;
+ unlink $filename ;
+
+ # Enable duplicate records
+ $DB_BTREE->{'flags'} = R_DUP ;
+
+ $x = tie %h, "DB_File", $filename, O_RDWR|O_CREAT, 0666, $DB_BTREE
+ or die "Cannot open $filename: $!\n";
+
+ # Add some key/value pairs to the file
+ $h{'Wall'} = 'Larry' ;
+ $h{'Wall'} = 'Brick' ; # Note the duplicate key
+ $h{'Wall'} = 'Brick' ; # Note the duplicate key and value
+ $h{'Smith'} = 'John' ;
+ $h{'mouse'} = 'mickey' ;
+
+ # iterate through the btree using seq
+ # and print each key/value pair.
+ $key = $value = 0 ;
+ for ($status = $x->seq($key, $value, R_FIRST) ;
+ $status == 0 ;
+ $status = $x->seq($key, $value, R_NEXT) )
+ { print "$key -> $value\n" }
+
+ undef $x ;
+ untie %h ;
+
+that prints:
+
+ Smith -> John
+ Wall -> Brick
+ Wall -> Brick
+ Wall -> Larry
+ mouse -> mickey
+
+This time we have got all the key/value pairs, including the multiple
+values associated with the key C.
+
+To make life easier when dealing with duplicate keys, B comes with
+a few utility methods.
+
+=head2 The get_dup() Method
+
+The C method assists in
+reading duplicate values from BTREE databases. The method can take the
+following forms:
+
+ $count = $x->get_dup($key) ;
+ @list = $x->get_dup($key) ;
+ %list = $x->get_dup($key, 1) ;
+
+In a scalar context the method returns the number of values associated
+with the key, C<$key>.
+
+In list context, it returns all the values which match C<$key>. Note
+that the values will be returned in an apparently random order.
+
+In list context, if the second parameter is present and evaluates
+TRUE, the method returns an associative array. The keys of the
+associative array correspond to the values that matched in the BTREE
+and the values of the array are a count of the number of times that
+particular value occurred in the BTREE.
+
+So assuming the database created above, we can use C like
+this:
+
+ use warnings ;
+ use strict ;
+ use DB_File ;
+
+ my ($filename, $x, %h) ;
+
+ $filename = "tree" ;
+
+ # Enable duplicate records
+ $DB_BTREE->{'flags'} = R_DUP ;
+
+ $x = tie %h, "DB_File", $filename, O_RDWR|O_CREAT, 0666, $DB_BTREE
+ or die "Cannot open $filename: $!\n";
+
+ my $cnt = $x->get_dup("Wall") ;
+ print "Wall occurred $cnt times\n" ;
+
+ my %hash = $x->get_dup("Wall", 1) ;
+ print "Larry is there\n" if $hash{'Larry'} ;
+ print "There are $hash{'Brick'} Brick Walls\n" ;
+
+ my @list = sort $x->get_dup("Wall") ;
+ print "Wall => [@list]\n" ;
+
+ @list = $x->get_dup("Smith") ;
+ print "Smith => [@list]\n" ;
+
+ @list = $x->get_dup("Dog") ;
+ print "Dog => [@list]\n" ;
+
+
+and it will print:
+
+ Wall occurred 3 times
+ Larry is there
+ There are 2 Brick Walls
+ Wall => [Brick Brick Larry]
+ Smith => [John]
+ Dog => []
+
+=head2 The find_dup() Method
+
+ $status = $X->find_dup($key, $value) ;
+
+This method checks for the existence of a specific key/value pair. If the
+pair exists, the cursor is left pointing to the pair and the method
+returns 0. Otherwise the method returns a non-zero value.
+
+Assuming the database from the previous example:
+
+ use warnings ;
+ use strict ;
+ use DB_File ;
+
+ my ($filename, $x, %h, $found) ;
+
+ $filename = "tree" ;
+
+ # Enable duplicate records
+ $DB_BTREE->{'flags'} = R_DUP ;
+
+ $x = tie %h, "DB_File", $filename, O_RDWR|O_CREAT, 0666, $DB_BTREE
+ or die "Cannot open $filename: $!\n";
+
+ $found = ( $x->find_dup("Wall", "Larry") == 0 ? "" : "not") ;
+ print "Larry Wall is $found there\n" ;
+
+ $found = ( $x->find_dup("Wall", "Harry") == 0 ? "" : "not") ;
+ print "Harry Wall is $found there\n" ;
+
+ undef $x ;
+ untie %h ;
+
+prints this
+
+ Larry Wall is there
+ Harry Wall is not there
+
+
+=head2 The del_dup() Method
+
+ $status = $X->del_dup($key, $value) ;
+
+This method deletes a specific key/value pair. It returns
+0 if they exist and have been deleted successfully.
+Otherwise the method returns a non-zero value.
+
+Again assuming the existence of the C database
+
+ use warnings ;
+ use strict ;
+ use DB_File ;
+
+ my ($filename, $x, %h, $found) ;
+
+ $filename = "tree" ;
+
+ # Enable duplicate records
+ $DB_BTREE->{'flags'} = R_DUP ;
+
+ $x = tie %h, "DB_File", $filename, O_RDWR|O_CREAT, 0666, $DB_BTREE
+ or die "Cannot open $filename: $!\n";
+
+ $x->del_dup("Wall", "Larry") ;
+
+ $found = ( $x->find_dup("Wall", "Larry") == 0 ? "" : "not") ;
+ print "Larry Wall is $found there\n" ;
+
+ undef $x ;
+ untie %h ;
+
+prints this
+
+ Larry Wall is not there
+
+=head2 Matching Partial Keys
+
+The BTREE interface has a feature which allows partial keys to be
+matched. This functionality is I available when the C method
+is used along with the R_CURSOR flag.
+
+ $x->seq($key, $value, R_CURSOR) ;
+
+Here is the relevant quote from the dbopen man page where it defines
+the use of the R_CURSOR flag with seq:
+
+ Note, for the DB_BTREE access method, the returned key is not
+ necessarily an exact match for the specified key. The returned key
+ is the smallest key greater than or equal to the specified key,
+ permitting partial key matches and range searches.
+
+In the example script below, the C sub uses this feature to find
+and print the first matching key/value pair given a partial key.
+
+ use warnings ;
+ use strict ;
+ use DB_File ;
+ use Fcntl ;
+
+ my ($filename, $x, %h, $st, $key, $value) ;
+
+ sub match
+ {
+ my $key = shift ;
+ my $value = 0;
+ my $orig_key = $key ;
+ $x->seq($key, $value, R_CURSOR) ;
+ print "$orig_key\t-> $key\t-> $value\n" ;
+ }
+
+ $filename = "tree" ;
+ unlink $filename ;
+
+ $x = tie %h, "DB_File", $filename, O_RDWR|O_CREAT, 0666, $DB_BTREE
+ or die "Cannot open $filename: $!\n";
+
+ # Add some key/value pairs to the file
+ $h{'mouse'} = 'mickey' ;
+ $h{'Wall'} = 'Larry' ;
+ $h{'Walls'} = 'Brick' ;
+ $h{'Smith'} = 'John' ;
+
+
+ $key = $value = 0 ;
+ print "IN ORDER\n" ;
+ for ($st = $x->seq($key, $value, R_FIRST) ;
+ $st == 0 ;
+ $st = $x->seq($key, $value, R_NEXT) )
+
+ { print "$key -> $value\n" }
+
+ print "\nPARTIAL MATCH\n" ;
+
+ match "Wa" ;
+ match "A" ;
+ match "a" ;
+
+ undef $x ;
+ untie %h ;
+
+Here is the output:
+
+ IN ORDER
+ Smith -> John
+ Wall -> Larry
+ Walls -> Brick
+ mouse -> mickey
+
+ PARTIAL MATCH
+ Wa -> Wall -> Larry
+ A -> Smith -> John
+ a -> mouse -> mickey
+
+=head1 DB_RECNO
+
+DB_RECNO provides an interface to flat text files. Both variable and
+fixed length records are supported.
+
+In order to make RECNO more compatible with Perl, the array offset for
+all RECNO arrays begins at 0 rather than 1 as in Berkeley DB.
+
+As with normal Perl arrays, a RECNO array can be accessed using
+negative indexes. The index -1 refers to the last element of the array,
+-2 the second last, and so on. Attempting to access an element before
+the start of the array will raise a fatal run-time error.
+
+=head2 The 'bval' Option
+
+The operation of the bval option warrants some discussion. Here is the
+definition of bval from the Berkeley DB 1.85 recno manual page:
+
+ The delimiting byte to be used to mark the end of a
+ record for variable-length records, and the pad charac-
+ ter for fixed-length records. If no value is speci-
+ fied, newlines (``\n'') are used to mark the end of
+ variable-length records and fixed-length records are
+ padded with spaces.
+
+The second sentence is wrong. In actual fact bval will only default to
+C<"\n"> when the openinfo parameter in dbopen is NULL. If a non-NULL
+openinfo parameter is used at all, the value that happens to be in bval
+will be used. That means you always have to specify bval when making
+use of any of the options in the openinfo parameter. This documentation
+error will be fixed in the next release of Berkeley DB.
+
+That clarifies the situation with regards Berkeley DB itself. What
+about B? Well, the behavior defined in the quote above is
+quite useful, so B conforms to it.
+
+That means that you can specify other options (e.g. cachesize) and
+still have bval default to C<"\n"> for variable length records, and
+space for fixed length records.
+
+Also note that the bval option only allows you to specify a single byte
+as a delimiter.
+
+=head2 A Simple Example
+
+Here is a simple example that uses RECNO (if you are using a version
+of Perl earlier than 5.004_57 this example won't work -- see
+L for a workaround).
+
+ use warnings ;
+ use strict ;
+ use DB_File ;
+
+ my $filename = "text" ;
+ unlink $filename ;
+
+ my @h ;
+ tie @h, "DB_File", $filename, O_RDWR|O_CREAT, 0666, $DB_RECNO
+ or die "Cannot open file 'text': $!\n" ;
+
+ # Add a few key/value pairs to the file
+ $h[0] = "orange" ;
+ $h[1] = "blue" ;
+ $h[2] = "yellow" ;
+
+ push @h, "green", "black" ;
+
+ my $elements = scalar @h ;
+ print "The array contains $elements entries\n" ;
+
+ my $last = pop @h ;
+ print "popped $last\n" ;
+
+ unshift @h, "white" ;
+ my $first = shift @h ;
+ print "shifted $first\n" ;
+
+ # Check for existence of a key
+ print "Element 1 Exists with value $h[1]\n" if $h[1] ;
+
+ # use a negative index
+ print "The last element is $h[-1]\n" ;
+ print "The 2nd last element is $h[-2]\n" ;
+
+ untie @h ;
+
+Here is the output from the script:
+
+ The array contains 5 entries
+ popped black
+ shifted white
+ Element 1 Exists with value blue
+ The last element is green
+ The 2nd last element is yellow
+
+=head2 Extra RECNO Methods
+
+If you are using a version of Perl earlier than 5.004_57, the tied
+array interface is quite limited. In the example script above
+C, C, C, C
+or determining the array length will not work with a tied array.
+
+To make the interface more useful for older versions of Perl, a number
+of methods are supplied with B to simulate the missing array
+operations. All these methods are accessed via the object returned from
+the tie call.
+
+Here are the methods:
+
+=over 5
+
+=item B<$X-Epush(list) ;>
+
+Pushes the elements of C to the end of the array.
+
+=item B<$value = $X-Epop ;>
+
+Removes and returns the last element of the array.
+
+=item B<$X-Eshift>
+
+Removes and returns the first element of the array.
+
+=item B<$X-Eunshift(list) ;>
+
+Pushes the elements of C to the start of the array.
+
+=item B<$X-Elength>
+
+Returns the number of elements in the array.
+
+=item B<$X-Esplice(offset, length, elements);>
+
+Returns a splice of the array.
+
+=back
+
+=head2 Another Example
+
+Here is a more complete example that makes use of some of the methods
+described above. It also makes use of the API interface directly (see
+L).
+
+ use warnings ;
+ use strict ;
+ my (@h, $H, $file, $i) ;
+ use DB_File ;
+ use Fcntl ;
+
+ $file = "text" ;
+
+ unlink $file ;
+
+ $H = tie @h, "DB_File", $file, O_RDWR|O_CREAT, 0666, $DB_RECNO
+ or die "Cannot open file $file: $!\n" ;
+
+ # first create a text file to play with
+ $h[0] = "zero" ;
+ $h[1] = "one" ;
+ $h[2] = "two" ;
+ $h[3] = "three" ;
+ $h[4] = "four" ;
+
+
+ # Print the records in order.
+ #
+ # The length method is needed here because evaluating a tied
+ # array in a scalar context does not return the number of
+ # elements in the array.
+
+ print "\nORIGINAL\n" ;
+ foreach $i (0 .. $H->length - 1) {
+ print "$i: $h[$i]\n" ;
+ }
+
+ # use the push & pop methods
+ $a = $H->pop ;
+ $H->push("last") ;
+ print "\nThe last record was [$a]\n" ;
+
+ # and the shift & unshift methods
+ $a = $H->shift ;
+ $H->unshift("first") ;
+ print "The first record was [$a]\n" ;
+
+ # Use the API to add a new record after record 2.
+ $i = 2 ;
+ $H->put($i, "Newbie", R_IAFTER) ;
+
+ # and a new record before record 1.
+ $i = 1 ;
+ $H->put($i, "New One", R_IBEFORE) ;
+
+ # delete record 3
+ $H->del(3) ;
+
+ # now print the records in reverse order
+ print "\nREVERSE\n" ;
+ for ($i = $H->length - 1 ; $i >= 0 ; -- $i)
+ { print "$i: $h[$i]\n" }
+
+ # same again, but use the API functions instead
+ print "\nREVERSE again\n" ;
+ my ($s, $k, $v) = (0, 0, 0) ;
+ for ($s = $H->seq($k, $v, R_LAST) ;
+ $s == 0 ;
+ $s = $H->seq($k, $v, R_PREV))
+ { print "$k: $v\n" }
+
+ undef $H ;
+ untie @h ;
+
+and this is what it outputs:
+
+ ORIGINAL
+ 0: zero
+ 1: one
+ 2: two
+ 3: three
+ 4: four
+
+ The last record was [four]
+ The first record was [zero]
+
+ REVERSE
+ 5: last
+ 4: three
+ 3: Newbie
+ 2: one
+ 1: New One
+ 0: first
+
+ REVERSE again
+ 5: last
+ 4: three
+ 3: Newbie
+ 2: one
+ 1: New One
+ 0: first
+
+Notes:
+
+=over 5
+
+=item 1.
+
+Rather than iterating through the array, C<@h> like this:
+
+ foreach $i (@h)
+
+it is necessary to use either this:
+
+ foreach $i (0 .. $H->length - 1)
+
+or this:
+
+ for ($a = $H->get($k, $v, R_FIRST) ;
+ $a == 0 ;
+ $a = $H->get($k, $v, R_NEXT) )
+
+=item 2.
+
+Notice that both times the C method was used the record index was
+specified using a variable, C<$i>, rather than the literal value
+itself. This is because C will return the record number of the
+inserted line via that parameter.
+
+=back
+
+=head1 THE API INTERFACE
+
+As well as accessing Berkeley DB using a tied hash or array, it is also
+possible to make direct use of most of the API functions defined in the
+Berkeley DB documentation.
+
+To do this you need to store a copy of the object returned from the tie.
+
+ $db = tie %hash, "DB_File", "filename" ;
+
+Once you have done that, you can access the Berkeley DB API functions
+as B methods directly like this:
+
+ $db->put($key, $value, R_NOOVERWRITE) ;
+
+B If you have saved a copy of the object returned from
+C, the underlying database file will I be closed until both
+the tied variable is untied and all copies of the saved object are
+destroyed.
+
+ use DB_File ;
+ $db = tie %hash, "DB_File", "filename"
+ or die "Cannot tie filename: $!" ;
+ ...
+ undef $db ;
+ untie %hash ;
+
+See L for more details.
+
+All the functions defined in L are available except for
+close() and dbopen() itself. The B method interface to the
+supported functions have been implemented to mirror the way Berkeley DB
+works whenever possible. In particular note that:
+
+=over 5
+
+=item *
+
+The methods return a status value. All return 0 on success.
+All return -1 to signify an error and set C<$!> to the exact
+error code. The return code 1 generally (but not always) means that the
+key specified did not exist in the database.
+
+Other return codes are defined. See below and in the Berkeley DB
+documentation for details. The Berkeley DB documentation should be used
+as the definitive source.
+
+=item *
+
+Whenever a Berkeley DB function returns data via one of its parameters,
+the equivalent B method does exactly the same.
+
+=item *
+
+If you are careful, it is possible to mix API calls with the tied
+hash/array interface in the same piece of code. Although only a few of
+the methods used to implement the tied interface currently make use of
+the cursor, you should always assume that the cursor has been changed
+any time the tied hash/array interface is used. As an example, this
+code will probably not do what you expect:
+
+ $X = tie %x, 'DB_File', $filename, O_RDWR|O_CREAT, 0777, $DB_BTREE
+ or die "Cannot tie $filename: $!" ;
+
+ # Get the first key/value pair and set the cursor
+ $X->seq($key, $value, R_FIRST) ;
+
+ # this line will modify the cursor
+ $count = scalar keys %x ;
+
+ # Get the second key/value pair.
+ # oops, it didn't, it got the last key/value pair!
+ $X->seq($key, $value, R_NEXT) ;
+
+The code above can be rearranged to get around the problem, like this:
+
+ $X = tie %x, 'DB_File', $filename, O_RDWR|O_CREAT, 0777, $DB_BTREE
+ or die "Cannot tie $filename: $!" ;
+
+ # this line will modify the cursor
+ $count = scalar keys %x ;
+
+ # Get the first key/value pair and set the cursor
+ $X->seq($key, $value, R_FIRST) ;
+
+ # Get the second key/value pair.
+ # worked this time.
+ $X->seq($key, $value, R_NEXT) ;
+
+=back
+
+All the constants defined in L for use in the flags parameters
+in the methods defined below are also available. Refer to the Berkeley
+DB documentation for the precise meaning of the flags values.
+
+Below is a list of the methods available.
+
+=over 5
+
+=item B<$status = $X-Eget($key, $value [, $flags]) ;>
+
+Given a key (C<$key>) this method reads the value associated with it
+from the database. The value read from the database is returned in the
+C<$value> parameter.
+
+If the key does not exist the method returns 1.
+
+No flags are currently defined for this method.
+
+=item B<$status = $X-Eput($key, $value [, $flags]) ;>
+
+Stores the key/value pair in the database.
+
+If you use either the R_IAFTER or R_IBEFORE flags, the C<$key> parameter
+will have the record number of the inserted key/value pair set.
+
+Valid flags are R_CURSOR, R_IAFTER, R_IBEFORE, R_NOOVERWRITE and
+R_SETCURSOR.
+
+=item B<$status = $X-Edel($key [, $flags]) ;>
+
+Removes all key/value pairs with key C<$key> from the database.
+
+A return code of 1 means that the requested key was not in the
+database.
+
+R_CURSOR is the only valid flag at present.
+
+=item B<$status = $X-Efd ;>
+
+Returns the file descriptor for the underlying database.
+
+See L for an explanation for why you should
+not use C to lock your database.
+
+=item B<$status = $X-Eseq($key, $value, $flags) ;>
+
+This interface allows sequential retrieval from the database. See
+L for full details.
+
+Both the C<$key> and C<$value> parameters will be set to the key/value
+pair read from the database.
+
+The flags parameter is mandatory. The valid flag values are R_CURSOR,
+R_FIRST, R_LAST, R_NEXT and R_PREV.
+
+=item B<$status = $X-Esync([$flags]) ;>
+
+Flushes any cached buffers to disk.
+
+R_RECNOSYNC is the only valid flag at present.
+
+=back
+
+=head1 DBM FILTERS
+
+A DBM Filter is a piece of code that is be used when you I
+want to make the same transformation to all keys and/or values in a
+DBM database.
+
+There are four methods associated with DBM Filters. All work identically,
+and each is used to install (or uninstall) a single DBM Filter. Each
+expects a single parameter, namely a reference to a sub. The only
+difference between them is the place that the filter is installed.
+
+To summarise:
+
+=over 5
+
+=item B
+
+If a filter has been installed with this method, it will be invoked
+every time you write a key to a DBM database.
+
+=item B
+
+If a filter has been installed with this method, it will be invoked
+every time you write a value to a DBM database.
+
+
+=item B
+
+If a filter has been installed with this method, it will be invoked
+every time you read a key from a DBM database.
+
+=item B
+
+If a filter has been installed with this method, it will be invoked
+every time you read a value from a DBM database.
+
+=back
+
+You can use any combination of the methods, from none, to all four.
+
+All filter methods return the existing filter, if present, or C
+in not.
+
+To delete a filter pass C to it.
+
+=head2 The Filter
+
+When each filter is called by Perl, a local copy of C<$_> will contain
+the key or value to be filtered. Filtering is achieved by modifying
+the contents of C<$_>. The return code from the filter is ignored.
+
+=head2 An Example -- the NULL termination problem.
+
+Consider the following scenario. You have a DBM database
+that you need to share with a third-party C application. The C application
+assumes that I keys and values are NULL terminated. Unfortunately
+when Perl writes to DBM databases it doesn't use NULL termination, so
+your Perl application will have to manage NULL termination itself. When
+you write to the database you will have to use something like this:
+
+ $hash{"$key\0"} = "$value\0" ;
+
+Similarly the NULL needs to be taken into account when you are considering
+the length of existing keys/values.
+
+It would be much better if you could ignore the NULL terminations issue
+in the main application code and have a mechanism that automatically
+added the terminating NULL to all keys and values whenever you write to
+the database and have them removed when you read from the database. As I'm
+sure you have already guessed, this is a problem that DBM Filters can
+fix very easily.
+
+ use warnings ;
+ use strict ;
+ use DB_File ;
+
+ my %hash ;
+ my $filename = "filt" ;
+ unlink $filename ;
+
+ my $db = tie %hash, 'DB_File', $filename, O_CREAT|O_RDWR, 0666, $DB_HASH
+ or die "Cannot open $filename: $!\n" ;
+
+ # Install DBM Filters
+ $db->filter_fetch_key ( sub { s/\0$// } ) ;
+ $db->filter_store_key ( sub { $_ .= "\0" } ) ;
+ $db->filter_fetch_value( sub { s/\0$// } ) ;
+ $db->filter_store_value( sub { $_ .= "\0" } ) ;
+
+ $hash{"abc"} = "def" ;
+ my $a = $hash{"ABC"} ;
+ # ...
+ undef $db ;
+ untie %hash ;
+
+Hopefully the contents of each of the filters should be
+self-explanatory. Both "fetch" filters remove the terminating NULL,
+and both "store" filters add a terminating NULL.
+
+
+=head2 Another Example -- Key is a C int.
+
+Here is another real-life example. By default, whenever Perl writes to
+a DBM database it always writes the key and value as strings. So when
+you use this:
+
+ $hash{12345} = "something" ;
+
+the key 12345 will get stored in the DBM database as the 5 byte string
+"12345". If you actually want the key to be stored in the DBM database
+as a C int, you will have to use C when writing, and C
+when reading.
+
+Here is a DBM Filter that does it:
+
+ use warnings ;
+ use strict ;
+ use DB_File ;
+ my %hash ;
+ my $filename = "filt" ;
+ unlink $filename ;
+
+
+ my $db = tie %hash, 'DB_File', $filename, O_CREAT|O_RDWR, 0666, $DB_HASH
+ or die "Cannot open $filename: $!\n" ;
+
+ $db->filter_fetch_key ( sub { $_ = unpack("i", $_) } ) ;
+ $db->filter_store_key ( sub { $_ = pack ("i", $_) } ) ;
+ $hash{123} = "def" ;
+ # ...
+ undef $db ;
+ untie %hash ;
+
+This time only two filters have been used -- we only need to manipulate
+the contents of the key, so it wasn't necessary to install any value
+filters.
+
+=head1 HINTS AND TIPS
+
+
+=head2 Locking: The Trouble with fd
+
+Until version 1.72 of this module, the recommended technique for locking
+B databases was to flock the filehandle returned from the "fd"
+function. Unfortunately this technique has been shown to be fundamentally
+flawed (Kudos to David Harris for tracking this down). Use it at your own
+peril!
+
+The locking technique went like this.
+
+ $db = tie(%db, 'DB_File', 'foo.db', O_CREAT|O_RDWR, 0644)
+ || die "dbcreat foo.db $!";
+ $fd = $db->fd;
+ open(DB_FH, "+<&=$fd") || die "dup $!";
+ flock (DB_FH, LOCK_EX) || die "flock: $!";
+ ...
+ $db{"Tom"} = "Jerry" ;
+ ...
+ flock(DB_FH, LOCK_UN);
+ undef $db;
+ untie %db;
+ close(DB_FH);
+
+In simple terms, this is what happens:
+
+=over 5
+
+=item 1.
+
+Use "tie" to open the database.
+
+=item 2.
+
+Lock the database with fd & flock.
+
+=item 3.
+
+Read & Write to the database.
+
+=item 4.
+
+Unlock and close the database.
+
+=back
+
+Here is the crux of the problem. A side-effect of opening the B
+database in step 2 is that an initial block from the database will get
+read from disk and cached in memory.
+
+To see why this is a problem, consider what can happen when two processes,
+say "A" and "B", both want to update the same B database
+using the locking steps outlined above. Assume process "A" has already
+opened the database and has a write lock, but it hasn't actually updated
+the database yet (it has finished step 2, but not started step 3 yet). Now
+process "B" tries to open the same database - step 1 will succeed,
+but it will block on step 2 until process "A" releases the lock. The
+important thing to notice here is that at this point in time both
+processes will have cached identical initial blocks from the database.
+
+Now process "A" updates the database and happens to change some of the
+data held in the initial buffer. Process "A" terminates, flushing
+all cached data to disk and releasing the database lock. At this point
+the database on disk will correctly reflect the changes made by process
+"A".
+
+With the lock released, process "B" can now continue. It also updates the
+database and unfortunately it too modifies the data that was in its
+initial buffer. Once that data gets flushed to disk it will overwrite
+some/all of the changes process "A" made to the database.
+
+The result of this scenario is at best a database that doesn't contain
+what you expect. At worst the database will corrupt.
+
+The above won't happen every time competing process update the same
+B database, but it does illustrate why the technique should
+not be used.
+
+=head2 Safe ways to lock a database
+
+Starting with version 2.x, Berkeley DB has internal support for locking.
+The companion module to this one, B, provides an interface
+to this locking functionality. If you are serious about locking
+Berkeley DB databases, I strongly recommend using B.
+
+If using B isn't an option, there are a number of modules
+available on CPAN that can be used to implement locking. Each one
+implements locking differently and has different goals in mind. It is
+therefore worth knowing the difference, so that you can pick the right
+one for your application. Here are the three locking wrappers:
+
+=over 5
+
+=item B
+
+A B wrapper which creates copies of the database file for
+read access, so that you have a kind of a multiversioning concurrent read
+system. However, updates are still serial. Use for databases where reads
+may be lengthy and consistency problems may occur.
+
+=item B
+
+A B wrapper that has the ability to lock and unlock the database
+while it is being used. Avoids the tie-before-flock problem by simply
+re-tie-ing the database when you get or drop a lock. Because of the
+flexibility in dropping and re-acquiring the lock in the middle of a
+session, this can be massaged into a system that will work with long
+updates and/or reads if the application follows the hints in the POD
+documentation.
+
+=item B
+
+An extremely lightweight B wrapper that simply flocks a lockfile
+before tie-ing the database and drops the lock after the untie. Allows
+one to use the same lockfile for multiple databases to avoid deadlock
+problems, if desired. Use for databases where updates are reads are
+quick and simple flock locking semantics are enough.
+
+=back
+
+=head2 Sharing Databases With C Applications
+
+There is no technical reason why a Berkeley DB database cannot be
+shared by both a Perl and a C application.
+
+The vast majority of problems that are reported in this area boil down
+to the fact that C strings are NULL terminated, whilst Perl strings are
+not. See L for a generic way to work around this problem.
+
+Here is a real example. Netscape 2.0 keeps a record of the locations you
+visit along with the time you last visited them in a DB_HASH database.
+This is usually stored in the file F<~/.netscape/history.db>. The key
+field in the database is the location string and the value field is the
+time the location was last visited stored as a 4 byte binary value.
+
+If you haven't already guessed, the location string is stored with a
+terminating NULL. This means you need to be careful when accessing the
+database.
+
+Here is a snippet of code that is loosely based on Tom Christiansen's
+I script (available from your nearest CPAN archive in
+F).
+
+ use warnings ;
+ use strict ;
+ use DB_File ;
+ use Fcntl ;
+
+ my ($dotdir, $HISTORY, %hist_db, $href, $binary_time, $date) ;
+ $dotdir = $ENV{HOME} || $ENV{LOGNAME};
+
+ $HISTORY = "$dotdir/.netscape/history.db";
+
+ tie %hist_db, 'DB_File', $HISTORY
+ or die "Cannot open $HISTORY: $!\n" ;;
+
+ # Dump the complete database
+ while ( ($href, $binary_time) = each %hist_db ) {
+
+ # remove the terminating NULL
+ $href =~ s/\x00$// ;
+
+ # convert the binary time into a user friendly string
+ $date = localtime unpack("V", $binary_time);
+ print "$date $href\n" ;
+ }
+
+ # check for the existence of a specific key
+ # remember to add the NULL
+ if ( $binary_time = $hist_db{"http://mox.perl.com/\x00"} ) {
+ $date = localtime unpack("V", $binary_time) ;
+ print "Last visited mox.perl.com on $date\n" ;
+ }
+ else {
+ print "Never visited mox.perl.com\n"
+ }
+
+ untie %hist_db ;
+
+=head2 The untie() Gotcha
+
+If you make use of the Berkeley DB API, it is I strongly
+recommended that you read L.
+
+Even if you don't currently make use of the API interface, it is still
+worth reading it.
+
+Here is an example which illustrates the problem from a B
+perspective:
+
+ use DB_File ;
+ use Fcntl ;
+
+ my %x ;
+ my $X ;
+
+ $X = tie %x, 'DB_File', 'tst.fil' , O_RDWR|O_TRUNC
+ or die "Cannot tie first time: $!" ;
+
+ $x{123} = 456 ;
+
+ untie %x ;
+
+ tie %x, 'DB_File', 'tst.fil' , O_RDWR|O_CREAT
+ or die "Cannot tie second time: $!" ;
+
+ untie %x ;
+
+When run, the script will produce this error message:
+
+ Cannot tie second time: Invalid argument at bad.file line 14.
+
+Although the error message above refers to the second tie() statement
+in the script, the source of the problem is really with the untie()
+statement that precedes it.
+
+Having read L you will probably have already guessed that the
+error is caused by the extra copy of the tied object stored in C<$X>.
+If you haven't, then the problem boils down to the fact that the
+B destructor, DESTROY, will not be called until I
+references to the tied object are destroyed. Both the tied variable,
+C<%x>, and C<$X> above hold a reference to the object. The call to
+untie() will destroy the first, but C<$X> still holds a valid
+reference, so the destructor will not get called and the database file
+F will remain open. The fact that Berkeley DB then reports the
+attempt to open a database that is already open via the catch-all
+"Invalid argument" doesn't help.
+
+If you run the script with the C<-w> flag the error message becomes:
+
+ untie attempted while 1 inner references still exist at bad.file line 12.
+ Cannot tie second time: Invalid argument at bad.file line 14.
+
+which pinpoints the real problem. Finally the script can now be
+modified to fix the original problem by destroying the API object
+before the untie:
+
+ ...
+ $x{123} = 456 ;
+
+ undef $X ;
+ untie %x ;
+
+ $X = tie %x, 'DB_File', 'tst.fil' , O_RDWR|O_CREAT
+ ...
+
+
+=head1 COMMON QUESTIONS
+
+=head2 Why is there Perl source in my database?
+
+If you look at the contents of a database file created by DB_File,
+there can sometimes be part of a Perl script included in it.
+
+This happens because Berkeley DB uses dynamic memory to allocate
+buffers which will subsequently be written to the database file. Being
+dynamic, the memory could have been used for anything before DB
+malloced it. As Berkeley DB doesn't clear the memory once it has been
+allocated, the unused portions will contain random junk. In the case
+where a Perl script gets written to the database, the random junk will
+correspond to an area of dynamic memory that happened to be used during
+the compilation of the script.
+
+Unless you don't like the possibility of there being part of your Perl
+scripts embedded in a database file, this is nothing to worry about.
+
+=head2 How do I store complex data structures with DB_File?
+
+Although B cannot do this directly, there is a module which
+can layer transparently over B to accomplish this feat.
+
+Check out the MLDBM module, available on CPAN in the directory
+F.
+
+=head2 What does "Invalid Argument" mean?
+
+You will get this error message when one of the parameters in the
+C call is wrong. Unfortunately there are quite a few parameters to
+get wrong, so it can be difficult to figure out which one it is.
+
+Here are a couple of possibilities:
+
+=over 5
+
+=item 1.
+
+Attempting to reopen a database without closing it.
+
+=item 2.
+
+Using the O_WRONLY flag.
+
+=back
+
+=head2 What does "Bareword 'DB_File' not allowed" mean?
+
+You will encounter this particular error message when you have the
+C pragma (or the full strict pragma) in your script.
+Consider this script:
+
+ use warnings ;
+ use strict ;
+ use DB_File ;
+ my %x ;
+ tie %x, DB_File, "filename" ;
+
+Running it produces the error in question:
+
+ Bareword "DB_File" not allowed while "strict subs" in use
+
+To get around the error, place the word C in either single or
+double quotes, like this:
+
+ tie %x, "DB_File", "filename" ;
+
+Although it might seem like a real pain, it is really worth the effort
+of having a C