diff --git a/steven/steven_reading_list.bib b/steven/steven_reading_list.bib new file mode 100644 index 0000000..3db89a5 --- /dev/null +++ b/steven/steven_reading_list.bib @@ -0,0 +1,208 @@ +@article{Weng2009, +abstract = {Development imposes great challenges. Internal ldquocorticalrdquorepresentations must be autonomously generated from interactive experiences. The eventual quality of these developed representations is of course important. Additionally, learning must be as fast as possible-to quickly derive better representation from limited experiences. Those who achieve both of these will have competitive advantages. We present a cortex-inspired theory called lobe component analysis (LCA) guided by the aforementioned dual criteria. A lobe component represents a high concentration of probability density of the neuronal input space. We explain how lobe components can achieve a dual-spatiotemporal (ldquobestrdquo and ldquofastestrdquo)-optimality, through mathematical analysis, in which we describe how lobe components plasticity can be temporally scheduled to take into account the history of observations in the best possible way. This contrasts with using only the last observation in gradient-based adaptive learning algorithms. Since they are based on two cell-centered mechanisms-Hebbian learning and lateral inhibition-lobe components develop in-place, meaning every networked neuron is individually responsible for the learning of its signal-processing characteristics within its connected network environment. There is no need for a separate learning network. We argue that in-place learning algorithms will be crucial for real-world large-size developmental applications due to their simplicity, low computational complexity, and generality. Our experimental results show that the learning speed of the LCA algorithm is drastically faster than other Hebbian-based updating methods and independent component analysis algorithms, thanks to its dual optimality, and it does not need to use any second- or higher order statistics. We also introduce the new principle of fast learning from stable representation.}, +author = {Weng, Juyang and Luciw, Matthew}, +doi = {10.1109/TAMD.2009.2021698}, +file = {:Users/GalaxyH/Documents/Mendeley Desktop/Weng, Luciw/IEEE Transactions on Autonomous Mental Development/Weng, Luciw - 2009 - Dually optimal neuronal layers Lobe component analysis.pdf:pdf}, +issn = {19430604}, +journal = {IEEE Transactions on Autonomous Mental Development}, +keywords = {Blind source separation,Cortical models,Feature extraction,Hebbian learning,Optimality,Plasticity}, +number = {1}, +pages = {68--85}, +title = {{Dually optimal neuronal layers: Lobe component analysis}}, +volume = {1}, +year = {2009} +} +@article{Weng2013, +author = {Weng, Juyang and Luciw, Matthew D. and Zhang, Qi}, +doi = {10.1109/TAMD.2013.2258398}, +file = {:Users/GalaxyH/Documents/Mendeley Desktop/Weng, Luciw, Zhang/IEEE Transactions on Autonomous Mental Development/Weng, Luciw, Zhang - 2013 - Brain-like emergent temporal processing Emergent open states.pdf:pdf}, +issn = {19430604}, +journal = {IEEE Transactions on Autonomous Mental Development}, +keywords = {Attention,behavior,brain-mind architecture,cognition,complexity,computer vision,perception,regression,representation,sequential abstraction,text understanding,time warping,transfer}, +number = {2}, +pages = {89--116}, +title = {{Brain-like emergent temporal processing: Emergent open states}}, +volume = {5}, +year = {2013} +} +@article{Weng2008, +abstract = {Currently, there is a lack of general-purpose in-place learning networks that model feature layers in the cortex. By "general-purpose" we mean a general yet adaptive high-dimensional function approximator. In-place learning is a biological concept rooted in the genomic equivalence principle, meaning that each neuron is fully responsible for its own learning in its environment and there is no need for an external learner. Presented in this paper is the Multilayer In-place Learning Network (MILN) for this ambitious goal. Computationally, in-place learning provides unusually efficient learning algorithms whose simplicity, low computational complexity, and generality are set apart from typical conventional learning algorithms. Based on the neuroscience literature, we model the layer 4 and layer 2/3 as the feature layers in the 6-layer laminar cortex, with layer 4 using unsupervised learning and layer 2/3 using supervised learning. As a necessary requirement for autonomous mental development, MILN generates invariant neurons in different layers, with increasing invariance from earlier to later layers and the total invariance in the last motor layer. Such self-generated invariant representation is enabled mainly by descending (top-down) connections. The self-generated invariant representation is used as intermediate representations for learning later tasks in open-ended development. ?? 2008 Elsevier Ltd. All rights reserved.}, +author = {Weng, Juyang and Luwang, Tianyu and Lu, Hong and Xue, Xiangyang}, +doi = {10.1016/j.neunet.2007.12.048}, +file = {:Users/GalaxyH/Documents/Mendeley Desktop/Weng et al/Neural Networks/Weng et al. - 2008 - Multilayer in-place learning networks for modeling functional layers in the laminar cortex.pdf:pdf}, +issn = {08936080}, +journal = {Neural Networks}, +keywords = {Abstraction,Cortex structure,Feature extraction,Laminar cortex,Multi-task learning,Regression,Representation,Supervised learning,Unsupervised learning}, +number = {2-3}, +pages = {150--159}, +pmid = {18314307}, +title = {{Multilayer in-place learning networks for modeling functional layers in the laminar cortex}}, +volume = {21}, +year = {2008} +} +@article{Weng2009a, +abstract = {It is largely unknown how the brain deals with time. Hidden Markov model (HMM) has a probability based mechanism to deal with time warping, but no effective online method exists that can deal with general active temporal abstraction. By online, we mean that the agent must respond to spatial and temporal context immediately while a sensory stream flows in. By general active temporal context, we mean active (learned) attention selects desirable temporal subsets within a dynamic length of recent history (e.g., beyond bigrams and trigrams). By temporal abstraction, we mean using abstract meaning of context, supervised at the motor end, instead of iconic forms. This paper reports four experiments of complex text processing using the framework of a general-purpose developmental spatiotemporal agent called Temporal Context Machines (TCM), demonstrating its power of forming online, active, abstract, temporal contexts. We show that it perfectly (100\%) solved a hypothetic problem called New Sentence Problem - after the TCM has learned synonyms under the corresponding contexts, it successfully recognized all possible new sentences (formed from the synonyms) that it has not learned. We show the TCM dealt with the Word Sense Disambiguation Problem where words are ambiguous without context. TCMs were also applied to the Part-of-Speech Problem, where the part of speech of the words in English language is identified according to contexts. In the fourth experiment, TCMs were employed to deal with the challenging Chunking Problem, in which subsequences of words are grouped and classified according to English linguistic units.}, +author = {Weng, Juyang and Zhang, Qi and Chi, Mingmin and Xue, Xiangyang}, +doi = {10.1109/DEVLRN.2009.5175540}, +file = {:Users/GalaxyH/Documents/Mendeley Desktop/Weng et al/2009 IEEE 8th International Conference on Development and Learning, ICDL 2009/Weng et al. - 2009 - Complex text processing by the temporal context machines.pdf:pdf}, +isbn = {9781424441181}, +journal = {2009 IEEE 8th International Conference on Development and Learning, ICDL 2009}, +pages = {1--8}, +title = {{Complex text processing by the temporal context machines}}, +year = {2009} +} +@article{Frome2013, +abstract = {Modern visual recognition systems are often limited in their ability to scale to large numbers of object categories. This limitation is in part due to the increasing difficulty of acquiring sufficient training data in the form of labeled images as the number of object categories grows. One remedy is to leverage data from other sources – such as text data – both to train visual models and to constrain their pre- dictions. In this paper we present a new deep visual-semantic embedding model trained to identify visual objects using both labeled image data as well as seman- tic information gleaned from unannotated text. We demonstrate that this model matches state-of-the-art performance on the 1000-class ImageNet object recogni- tion challenge while making more semantically reasonable errors, and also show that the semantic information can be exploited to make predictions about tens of thousands of image labels not observed during training. Semantic knowledge improves such zero-shot predictions achieving hit rates of up to 18\% across thou- sands of novel labels never seen by the visual model.}, +author = {Frome, Andrea and Corrado, Gs and Shlens, Jonathon}, +file = {:Users/GalaxyH/Documents/Mendeley Desktop/Frome, Corrado, Shlens/Advances in Neural Information Processing Systems/Frome, Corrado, Shlens - 2013 - Devise A deep visual-semantic embedding model.pdf:pdf}, +issn = {10495258}, +journal = {Advances in Neural Information Processing Systems}, +pages = {1--11}, +title = {{Devise: A deep visual-semantic embedding model}}, +url = {http://papers.nips.cc/paper/5204-devise-a-deep-visual-semantic-embedding-model}, +year = {2013} +} +@article{Norouzi2014, +abstract = {Several recent publications have proposed methods for mapping images into con- tinuous semantic embedding spaces. In some cases the embedding space is trained jointly with the image transformation. In other cases the semantic embedding space is established by an independent natural language processing task, and then the image transformation into that space is learned in a second stage. Proponents of these image embedding systems have stressed their advantages over the traditional n-way classification framing of image understanding, particularly in terms of the promise for zero-shot learning – the ability to correctly annotate images of previously unseen object categories. In this paper, we propose a simple method for constructing an image embedding system from any existing n-way image classifier and a semantic word embedding model, which contains the n class labels in its vocabulary. Our method maps images into the semantic embedding space via convex combination of the class label embedding vectors, and requires no addi- tional training. We show that this simple and direct method confers many of the advantages associated with more complex image embedding schemes, and indeed outperforms state of the art methods on the ImageNet zero-shot learning task.}, +archivePrefix = {arXiv}, +arxivId = {arXiv:1312.5650v3}, +author = {Norouzi, Mohammad and Mikolov, Tomas and Bengio, Samy and Singer, Yoram and Mar, L G}, +eprint = {arXiv:1312.5650v3}, +file = {:Users/GalaxyH/Documents/Mendeley Desktop/Norouzi et al/ArXiV/Norouzi et al. - 2014 - Zero-Shot Learning by Convex Combination of Semantic Embeddings.pdf:pdf}, +journal = {ArXiV}, +keywords = {0-shot learning by convex,combination of}, +pages = {1--9}, +title = {{Zero-Shot Learning by Convex Combination of Semantic Embeddings}}, +year = {2014} +} +@article{Schmidhuber2014, +abstract = {In recent years, deep artificial neural networks (including recurrent ones) have won numerous contests in pattern recognition and machine learning. This historical survey compactly summarises relevant work, much of it from the previous millennium. Shallow and deep learners are distinguished by the depth of their credit assignment paths, which are chains of possibly learnable, causal links between actions and effects. I review deep supervised learning (also recapitulating the history of backpropagation), unsupervised learning, reinforcement learning \& evolutionary computation, and indirect search for short programs encoding deep and large networks.}, +archivePrefix = {arXiv}, +arxivId = {1404.7828}, +author = {Schmidhuber, Juergen}, +eprint = {1404.7828}, +file = {:Users/GalaxyH/Documents/Mendeley Desktop/Schmidhuber/Unknown/Schmidhuber - 2014 - Deep Learning in Neural Networks An Overview.pdf:pdf}, +month = apr, +pages = {75}, +title = {{Deep Learning in Neural Networks: An Overview}}, +url = {http://arxiv.org/abs/1404.7828}, +year = {2014} +} +@article{Auli2013, +author = {Auli, Michael and Galley, Michel and Quirk, Chris and Zweig, Geoffrey}, +file = {:Users/GalaxyH/Documents/Mendeley Desktop/Auli et al/EMNLP/Auli et al. - 2013 - Joint Language and Translation Modeling with Recurrent Neural Networks.pdf:pdf}, +journal = {EMNLP}, +number = {October}, +pages = {1044--1054}, +title = {{Joint Language and Translation Modeling with Recurrent Neural Networks.}}, +url = {http://research.microsoft.com/en-us/um/people/gzweig/Pubs/EMNLP2013RNNMT.pdf}, +year = {2013} +} +@article{Bengio2003, +abstract = {A goal of statistical language modeling is to learn the joint probability function of sequences of words in a language. This is intrinsically difficult because of the curse of dimensionality: a word sequence on which the model will be tested is likely to be different from all the word sequences seen during training. Traditional but very successful approaches based on n-grams obtain generalization by concatenating very short overlapping sequences seen in the training set. We propose to fight the curse of dimensionality by learning a distributed representation for words which allows each training sentence to inform the model about an exponential number of semantically neighboring sentences. The model learns simultaneously (1) a distributed representation for each word along with (2) the probability function for word sequences, expressed in terms of these representations. Generalization is obtained because a sequence of words that has never been seen before gets high probability if it is made of words that are similar (in the sense of having a nearby representation) to words forming an already seen sentence. Training such large models (with millions of parameters) within a reasonable time is itself a significant challenge. We report on experiments using neural networks for the probability function, showing on two text corpora that the proposed approach significantly improves on state-of-the-art n-gram models, and that the proposed approach allows to take advantage of longer contexts.}, +author = {Bengio, Yoshua and Ducharme, R\'{e}jean and Vincent, Pascal and Janvin, Christian}, +doi = {10.1162/153244303322533223}, +file = {:Users/GalaxyH/Documents/Mendeley Desktop/Bengio et al/The Journal of Machine Learning Research/Bengio et al. - 2003 - A Neural Probabilistic Language Model.pdf:pdf}, +isbn = {1532-4435}, +issn = {15324435}, +journal = {The Journal of Machine Learning Research}, +keywords = {artificial neural networks,curse of dimensionality,distributed representation,statistical language modeling}, +pages = {1137--1155}, +title = {{A Neural Probabilistic Language Model}}, +volume = {3}, +year = {2003} +} +@article{Collobert2011, +archivePrefix = {arXiv}, +arxivId = {arXiv:1103.0398v1}, +author = {Collobert, Ronan}, +eprint = {arXiv:1103.0398v1}, +file = {:Users/GalaxyH/Documents/Mendeley Desktop/Collobert/Journal of Machine Learning Research/Collobert - 2011 - Natural Language Processing (Almost) from Scratch.pdf:pdf}, +issn = {1532-4435}, +journal = {Journal of Machine Learning Research}, +keywords = {Artificial intelligence,Natural language processin,part of speech tagging}, +pages = {2493--2537}, +title = {{Natural Language Processing (Almost) from Scratch}}, +url = {http://web.b.ebscohost.com/ehost/pdfviewer/pdfviewer?sid=45dd5244-0968-496b-bbdc-53e74576688a@sessionmgr114\&vid=1\&hid=125}, +volume = {12}, +year = {2011} +} +@article{Devlin2014, +abstract = {Recent work has shown success in us- ing neural network language models (NNLMs) as features in MT systems. Here, we present a novel formulation for a neural network joint model (NNJM), which augments the NNLM with a source context window. Our model is purely lexicalized and can be integrated into any MT decoder. We also present several variations of the NNJM which provide significant additive improvements. Although the model is quite simple, it yields strong empirical results. On the NIST OpenMT12 Arabic-English condition, the NNJM features produce a gain of +3.0 BLEU on top of a powerful, feature-rich baseline which already includes a target-only NNLM. The NNJM features also produce a gain of +6.3 BLEU on top of a simpler baseline equivalent to Chiang’s (2007) original Hiero implementation. Additionally, we describe two novel techniques for overcoming the historically high cost of using NNLM-style models in MT decoding. These techniques speed up NNJM computation by a factor of 10,000x, making the model as fast as a standard back-off LM.}, +author = {Devlin, Jacob and Zbib, Rabih and Huang, Zhongqiang and Lamar, Thomas and Schwartz, Richard and Makhoul, John}, +file = {:Users/GalaxyH/Documents/Mendeley Desktop/Devlin et al/52nd Annual Meeting of the Association for Computational Linguistics/Devlin et al. - 2014 - Fast and Robust Neural Network Joint Models for Statistical Machine Translation.pdf:pdf}, +journal = {52nd Annual Meeting of the Association for Computational Linguistics}, +pages = {1370--1380}, +title = {{Fast and Robust Neural Network Joint Models for Statistical Machine Translation}}, +url = {http://acl2014.org/acl2014/P14-1/pdf/P14-1129.pdf}, +year = {2014} +} +@article{Hu2014, +author = {Hu, Yuening and Auli, Michael and Gao, Q and Gao, Jianfeng}, +file = {:Users/GalaxyH/Documents/Mendeley Desktop/Hu et al/Proceedings of the 14th Conference of the \ldots/Hu et al. - 2014 - Minimum Translation Modeling with Recurrent Neural Networks.pdf:pdf}, +journal = {Proceedings of the 14th Conference of the \ldots}, +pages = {20--29}, +title = {{Minimum Translation Modeling with Recurrent Neural Networks}}, +url = {http://www.cs.umd.edu/~ynhu/publications/eacl2014\_rnn\_mtu.pdf}, +year = {2014} +} +@article{Huang2012, +abstract = {Unsupervised word representations are very useful in NLP tasks both as inputs to learning algorithms and as extra word features in NLP systems. However, most of these models are built with only local context and one representation per word. This is problematic}, +author = {Huang, Eric H and Socher, Richard and Manning, Christopher D and Ng, Andrew}, +file = {:Users/GalaxyH/Documents/Mendeley Desktop/Huang et al/Proceedings of the 50th Annual Meeting of the Association for Computational Linguistics Long Papers-Volume 1/Huang et al. - 2012 - Improving word representations via global context and multiple word prototypes.pdf:pdf}, +isbn = {9781937284244}, +journal = {Proceedings of the 50th Annual Meeting of the Association for Computational Linguistics: Long Papers-Volume 1}, +pages = {873--882}, +title = {{Improving word representations via global context and multiple word prototypes}}, +year = {2012} +} +@article{Koehn2007, +author = {Koehn, Philipp and Hoang, H and Birch, Alexandra}, +file = {:Users/GalaxyH/Documents/Mendeley Desktop/Koehn, Hoang, Birch/Proceedings of the 45th \ldots/Koehn, Hoang, Birch - 2007 - Moses Open source toolkit for statistical machine translation.pdf:pdf}, +journal = {Proceedings of the 45th \ldots}, +number = {June}, +pages = {177--180}, +title = {{Moses: Open source toolkit for statistical machine translation}}, +url = {http://dl.acm.org/citation.cfm?id=1557821}, +year = {2007} +} +@article{Mikolov2010, +abstract = {基于RNN模型的语言模型,详细可参考作者的博士论文。周期神经网络。但是上下文,也没用取全部的,只取到了前5个。}, +author = {Mikolov, T and Karafiat, M and Burget, L and Cernocky, J and Khudanpur, S}, +file = {:Users/GalaxyH/Documents/Mendeley Desktop/Mikolov et al/Interspeech/Mikolov et al. - 2010 - Recurrent Neural Network based Language Model.pdf:pdf}, +journal = {Interspeech}, +number = {September}, +pages = {1045--1048}, +title = {{Recurrent Neural Network based Language Model}}, +year = {2010} +} +@article{Mikolov2013, +abstract = {Continuous space language models have re- cently demonstrated outstanding results across a variety of tasks. In this paper, we ex- amine the vector-space word representations that are implicitly learned by the input-layer weights. We find that these representations are surprisingly good at capturing syntactic and semantic regularities in language, and that each relationship is characterized by a relation-specific vector offset. This allows vector-oriented reasoning based on the offsets between words. For example, the male/female relationship is automatically learned, and with the induced vector representations, “King - Man + Woman” results in a vector very close to “Queen.” We demonstrate that the word vectors capture syntactic regularities by means of syntactic analogy questions (provided with this paper), and are able to correctly answer almost 40\% of the questions. We demonstrate that the word vectors capture semantic regu- larities by using the vector offset method to answer SemEval-2012 Task 2 questions. Re- markably, this method outperforms the best previous systems. 1}, +author = {Mikolov, Tomas and Yih, Wen-tau and Zweig, Geoffrey}, +file = {:Users/GalaxyH/Documents/Mendeley Desktop/Mikolov, Yih, Zweig/Proceedings of NAACL-HLT/Mikolov, Yih, Zweig - 2013 - Linguistic regularities in continuous space word representations.pdf:pdf}, +journal = {Proceedings of NAACL-HLT}, +number = {June}, +pages = {746--751}, +title = {{Linguistic regularities in continuous space word representations}}, +url = {http://scholar.google.com/scholar?hl=en\&btnG=Search\&q=intitle:Linguistic+Regularities+in+Continuous+Space+Word+Representations\#0$\backslash$nhttps://www.aclweb.org/anthology/N/N13/N13-1090.pdf}, +year = {2013} +} +@article{Turian2010, +abstract = {If we take an existing supervised \{NLP\} system, a simple and general way to improve accuracy is to use unsupervised word representations as extra word features. We evaluate Brown clusters, Collobert and Weston (2008) embeddings, and \{HLBL\} (Mnih \& Hinton, 2009) embeddings of words on both \{NER\} and chunking. We use near state-of-the-art supervised baselines, and find that each of the three word representations improves the accuracy of these baselines. We find further improvements by combining different word representations. You can download our word features, for off-the-shelf use in existing \{NLP\} systems, as well as our code, here: http://metaoptimize.com/projects/wordreprs/}, +author = {Turian, Joseph and Ratinov, Lev and Bengio, Yoshua}, +file = {:Users/GalaxyH/Documents/Mendeley Desktop/Turian, Ratinov, Bengio/Proceedings of the 48th Annual Meeting of the Association for Computational Linguistics/Turian, Ratinov, Bengio - 2010 - Word Representations A Simple and General Method for Semi-supervised Learning.pdf:pdf}, +isbn = {9781617388088}, +journal = {Proceedings of the 48th Annual Meeting of the Association for Computational Linguistics}, +pages = {384--394}, +title = {{Word Representations: A Simple and General Method for Semi-supervised Learning}}, +url = {http://dl.acm.org/citation.cfm?id=1858721$\backslash$nhttp://dl.acm.org/citation.cfm?id=1858681.1858721}, +year = {2010} +} +@article{Zou2013, +abstract = {We introduce bilingual word embeddings: semantic embeddings associated across two languages in the context of neural language models. We propose a method to learn bilingual embeddings from a large unlabeled corpus, while utilizing MT word alignments to constrain translational equivalence. The new embeddings significantly out-perform baselines in word semantic similarity. A single semantic similarity feature induced with bilingual embeddings adds near half a BLEU point to the results of NIST08 Chinese-English machine translation task.}, +author = {Zou, Will Y and Socher, Richard and Cer, Daniel and Manning, Christopher D}, +file = {:Users/GalaxyH/Documents/Mendeley Desktop/Zou et al/Proceedings of the 2013 Conference on Empirical Methods in Natural Language Processing (EMNLP 2013)/Zou et al. - 2013 - Bilingual Word Embeddings for Phrase-Based Machine Translation.pdf:pdf}, +journal = {Proceedings of the 2013 Conference on Empirical Methods in Natural Language Processing (EMNLP 2013)}, +title = {{Bilingual Word Embeddings for Phrase-Based Machine Translation}}, +year = {2013} +} diff --git a/steven/steven_reading_list.pdf b/steven/steven_reading_list.pdf new file mode 100644 index 0000000..ccfab98 Binary files /dev/null and b/steven/steven_reading_list.pdf differ diff --git a/steven/steven_reading_list.tex b/steven/steven_reading_list.tex new file mode 100755 index 0000000..86109ca --- /dev/null +++ b/steven/steven_reading_list.tex @@ -0,0 +1,22 @@ +\documentclass[a4paper]{article} +% Preamble Area + +\usepackage{geometry} +\usepackage{graphicx} + + +\title{\textbf{Stevent's reading list}} +\author{Steven Yu-chun Huang} + + +\begin{document} +\maketitle + +\section{What I have read.} +I've been working on machine translation project, so I read both traditional statistical machine translation algorithms and word embeddings methods.\cite{Auli2013, Bengio2003, Collobert2011, Devlin2014, Hu2014, Huang2012, Koehn2007, Mikolov2010, Mikolov2013, Turian2010, Zou2013, Frome2013, Norouzi2014, Schmidhuber2014}. +I also spent some time implementing and testing "Developmental Mental Architecture" done by Weng \cite{Weng2008, Weng2009, Weng2009a, Weng2013}. + +\bibliographystyle{abbrv} +\bibliography{steven_reading_list} + +\end{document} \ No newline at end of file