@misc{13080, abstract = {Data for the manuscript 'Closing of the Induced Gap in a Hybrid Superconductor-Semiconductor Nanowire' ([2006.01275] Closing of the Induced Gap in a Hybrid Superconductor-Semiconductor Nanowire (arxiv.org)) We upload a pdf with extended data sets, and the raw data for these extended datasets as well.}, author = {Puglia, Denise and Martinez, Esteban and Menard, Gerbold and Pöschl, Andreas and Gronin, Sergei and Gardner, Geoffrey and Kallaher, Ray and Manfra, Michael and Marcus, Charles and Higginbotham, Andrew P and Casparis, Lucas}, publisher = {Zenodo}, title = {{Data for 'Closing of the Induced Gap in a Hybrid Superconductor-Semiconductor Nanowire}}, doi = {10.5281/ZENODO.4592435}, year = {2021}, } @inproceedings{13146, abstract = {A recent line of work has analyzed the theoretical properties of deep neural networks via the Neural Tangent Kernel (NTK). In particular, the smallest eigenvalue of the NTK has been related to the memorization capacity, the global convergence of gradient descent algorithms and the generalization of deep nets. However, existing results either provide bounds in the two-layer setting or assume that the spectrum of the NTK matrices is bounded away from 0 for multi-layer networks. In this paper, we provide tight bounds on the smallest eigenvalue of NTK matrices for deep ReLU nets, both in the limiting case of infinite widths and for finite widths. In the finite-width setting, the network architectures we consider are fairly general: we require the existence of a wide layer with roughly order of N neurons, N being the number of data samples; and the scaling of the remaining layer widths is arbitrary (up to logarithmic factors). To obtain our results, we analyze various quantities of independent interest: we give lower bounds on the smallest singular value of hidden feature matrices, and upper bounds on the Lipschitz constant of input-output feature maps.}, author = {Nguyen, Quynh and Mondelli, Marco and Montufar, Guido}, booktitle = {Proceedings of the 38th International Conference on Machine Learning}, isbn = {9781713845065}, issn = {2640-3498}, location = {Virtual}, pages = {8119--8129}, publisher = {ML Research Press}, title = {{Tight bounds on the smallest Eigenvalue of the neural tangent kernel for deep ReLU networks}}, volume = {139}, year = {2021}, } @inproceedings{13147, abstract = {We investigate fast and communication-efficient algorithms for the classic problem of minimizing a sum of strongly convex and smooth functions that are distributed among n different nodes, which can communicate using a limited number of bits. Most previous communication-efficient approaches for this problem are limited to first-order optimization, and therefore have \emph{linear} dependence on the condition number in their communication complexity. We show that this dependence is not inherent: communication-efficient methods can in fact have sublinear dependence on the condition number. For this, we design and analyze the first communication-efficient distributed variants of preconditioned gradient descent for Generalized Linear Models, and for Newton’s method. Our results rely on a new technique for quantizing both the preconditioner and the descent direction at each step of the algorithms, while controlling their convergence rate. We also validate our findings experimentally, showing faster convergence and reduced communication relative to previous methods.}, author = {Alimisis, Foivos and Davies, Peter and Alistarh, Dan-Adrian}, booktitle = {Proceedings of the 38th International Conference on Machine Learning}, isbn = {9781713845065}, issn = {2640-3498}, location = {Virtual}, pages = {196--206}, publisher = {ML Research Press}, title = {{Communication-efficient distributed optimization with quantized preconditioners}}, volume = {139}, year = {2021}, } @article{14117, abstract = {The two fields of machine learning and graphical causality arose and are developed separately. However, there is, now, cross-pollination and increasing interest in both fields to benefit from the advances of the other. In this article, we review fundamental concepts of causal inference and relate them to crucial open problems of machine learning, including transfer and generalization, thereby assaying how causality can contribute to modern machine learning research. This also applies in the opposite direction: we note that most work in causality starts from the premise that the causal variables are given. A central problem for AI and causality is, thus, causal representation learning, that is, the discovery of high-level causal variables from low-level observations. Finally, we delineate some implications of causality for machine learning and propose key research areas at the intersection of both communities.}, author = {Scholkopf, Bernhard and Locatello, Francesco and Bauer, Stefan and Ke, Nan Rosemary and Kalchbrenner, Nal and Goyal, Anirudh and Bengio, Yoshua}, issn = {1558-2256}, journal = {Proceedings of the IEEE}, keywords = {Electrical and Electronic Engineering}, number = {5}, pages = {612--634}, publisher = {Institute of Electrical and Electronics Engineers}, title = {{Toward causal representation learning}}, doi = {10.1109/jproc.2021.3058954}, volume = {109}, year = {2021}, } @inproceedings{14176, abstract = {Intensive care units (ICU) are increasingly looking towards machine learning for methods to provide online monitoring of critically ill patients. In machine learning, online monitoring is often formulated as a supervised learning problem. Recently, contrastive learning approaches have demonstrated promising improvements over competitive supervised benchmarks. These methods rely on well-understood data augmentation techniques developed for image data which do not apply to online monitoring. In this work, we overcome this limitation by supplementing time-series data augmentation techniques with a novel contrastive learning objective which we call neighborhood contrastive learning (NCL). Our objective explicitly groups together contiguous time segments from each patient while maintaining state-specific information. Our experiments demonstrate a marked improvement over existing work applying contrastive methods to medical time-series.}, author = {Yèche, Hugo and Dresdner, Gideon and Locatello, Francesco and Hüser, Matthias and Rätsch, Gunnar}, booktitle = {Proceedings of 38th International Conference on Machine Learning}, location = {Virtual}, pages = {11964--11974}, publisher = {ML Research Press}, title = {{Neighborhood contrastive learning applied to online patient monitoring}}, volume = {139}, year = {2021}, } @inproceedings{14177, abstract = {The focus of disentanglement approaches has been on identifying independent factors of variation in data. However, the causal variables underlying real-world observations are often not statistically independent. In this work, we bridge the gap to real-world scenarios by analyzing the behavior of the most prominent disentanglement approaches on correlated data in a large-scale empirical study (including 4260 models). We show and quantify that systematically induced correlations in the dataset are being learned and reflected in the latent representations, which has implications for downstream applications of disentanglement such as fairness. We also demonstrate how to resolve these latent correlations, either using weak supervision during training or by post-hoc correcting a pre-trained model with a small number of labels.}, author = {Träuble, Frederik and Creager, Elliot and Kilbertus, Niki and Locatello, Francesco and Dittadi, Andrea and Goyal, Anirudh and Schölkopf, Bernhard and Bauer, Stefan}, booktitle = {Proceedings of the 38th International Conference on Machine Learning}, location = {Virtual}, pages = {10401--10412}, publisher = {ML Research Press}, title = {{On disentangled representations learned from correlated data}}, volume = {139}, year = {2021}, } @inproceedings{14178, abstract = {Learning meaningful representations that disentangle the underlying structure of the data generating process is considered to be of key importance in machine learning. While disentangled representations were found to be useful for diverse tasks such as abstract reasoning and fair classification, their scalability and real-world impact remain questionable. We introduce a new high-resolution dataset with 1M simulated images and over 1,800 annotated real-world images of the same setup. In contrast to previous work, this new dataset exhibits correlations, a complex underlying structure, and allows to evaluate transfer to unseen simulated and real-world settings where the encoder i) remains in distribution or ii) is out of distribution. We propose new architectures in order to scale disentangled representation learning to realistic high-resolution settings and conduct a large-scale empirical study of disentangled representations on this dataset. We observe that disentanglement is a good predictor for out-of-distribution (OOD) task performance.}, author = {Dittadi, Andrea and Träuble, Frederik and Locatello, Francesco and Wüthrich, Manuel and Agrawal, Vaibhav and Winther, Ole and Bauer, Stefan and Schölkopf, Bernhard}, booktitle = {The Ninth International Conference on Learning Representations}, location = {Virtual}, title = {{On the transfer of disentangled representations in realistic settings}}, year = {2021}, } @inproceedings{14179, abstract = {Self-supervised representation learning has shown remarkable success in a number of domains. A common practice is to perform data augmentation via hand-crafted transformations intended to leave the semantics of the data invariant. We seek to understand the empirical success of this approach from a theoretical perspective. We formulate the augmentation process as a latent variable model by postulating a partition of the latent representation into a content component, which is assumed invariant to augmentation, and a style component, which is allowed to change. Unlike prior work on disentanglement and independent component analysis, we allow for both nontrivial statistical and causal dependencies in the latent space. We study the identifiability of the latent representation based on pairs of views of the observations and prove sufficient conditions that allow us to identify the invariant content partition up to an invertible mapping in both generative and discriminative settings. We find numerical simulations with dependent latent variables are consistent with our theory. Lastly, we introduce Causal3DIdent, a dataset of high-dimensional, visually complex images with rich causal dependencies, which we use to study the effect of data augmentations performed in practice.}, author = {Kügelgen, Julius von and Sharma, Yash and Gresele, Luigi and Brendel, Wieland and Schölkopf, Bernhard and Besserve, Michel and Locatello, Francesco}, booktitle = {Advances in Neural Information Processing Systems}, isbn = {9781713845393}, location = {Virtual}, pages = {16451--16467}, title = {{Self-supervised learning with data augmentations provably isolates content from style}}, volume = {34}, year = {2021}, } @inproceedings{14180, abstract = {Modern neural network architectures can leverage large amounts of data to generalize well within the training distribution. However, they are less capable of systematic generalization to data drawn from unseen but related distributions, a feat that is hypothesized to require compositional reasoning and reuse of knowledge. In this work, we present Neural Interpreters, an architecture that factorizes inference in a self-attention network as a system of modules, which we call \emph{functions}. Inputs to the model are routed through a sequence of functions in a way that is end-to-end learned. The proposed architecture can flexibly compose computation along width and depth, and lends itself well to capacity extension after training. To demonstrate the versatility of Neural Interpreters, we evaluate it in two distinct settings: image classification and visual abstract reasoning on Raven Progressive Matrices. In the former, we show that Neural Interpreters perform on par with the vision transformer using fewer parameters, while being transferrable to a new task in a sample efficient manner. In the latter, we find that Neural Interpreters are competitive with respect to the state-of-the-art in terms of systematic generalization. }, author = {Rahaman, Nasim and Gondal, Muhammad Waleed and Joshi, Shruti and Gehler, Peter and Bengio, Yoshua and Locatello, Francesco and Schölkopf, Bernhard}, booktitle = {Advances in Neural Information Processing Systems}, isbn = {9781713845393}, location = {Virtual}, pages = {10985--10998}, title = {{Dynamic inference with neural interpreters}}, volume = {34}, year = {2021}, } @inproceedings{14181, abstract = {Variational Inference makes a trade-off between the capacity of the variational family and the tractability of finding an approximate posterior distribution. Instead, Boosting Variational Inference allows practitioners to obtain increasingly good posterior approximations by spending more compute. The main obstacle to widespread adoption of Boosting Variational Inference is the amount of resources necessary to improve over a strong Variational Inference baseline. In our work, we trace this limitation back to the global curvature of the KL-divergence. We characterize how the global curvature impacts time and memory consumption, address the problem with the notion of local curvature, and provide a novel approximate backtracking algorithm for estimating local curvature. We give new theoretical convergence rates for our algorithms and provide experimental validation on synthetic and real-world datasets.}, author = {Dresdner, Gideon and Shekhar, Saurav and Pedregosa, Fabian and Locatello, Francesco and Rätsch, Gunnar}, booktitle = {Proceedings of the Thirtieth International Joint Conference on Artificial Intelligence}, location = {Montreal, Canada}, pages = {2337--2343}, publisher = {International Joint Conferences on Artificial Intelligence}, title = {{Boosting variational inference with locally adaptive step-sizes}}, doi = {10.24963/ijcai.2021/322}, year = {2021}, } @inproceedings{14182, abstract = {When machine learning systems meet real world applications, accuracy is only one of several requirements. In this paper, we assay a complementary perspective originating from the increasing availability of pre-trained and regularly improving state-of-the-art models. While new improved models develop at a fast pace, downstream tasks vary more slowly or stay constant. Assume that we have a large unlabelled data set for which we want to maintain accurate predictions. Whenever a new and presumably better ML models becomes available, we encounter two problems: (i) given a limited budget, which data points should be re-evaluated using the new model?; and (ii) if the new predictions differ from the current ones, should we update? Problem (i) is about compute cost, which matters for very large data sets and models. Problem (ii) is about maintaining consistency of the predictions, which can be highly relevant for downstream applications; our demand is to avoid negative flips, i.e., changing correct to incorrect predictions. In this paper, we formalize the Prediction Update Problem and present an efficient probabilistic approach as answer to the above questions. In extensive experiments on standard classification benchmark data sets, we show that our method outperforms alternative strategies along key metrics for backward-compatible prediction updates.}, author = {Träuble, Frederik and Kügelgen, Julius von and Kleindessner, Matthäus and Locatello, Francesco and Schölkopf, Bernhard and Gehler, Peter}, booktitle = {35th Conference on Neural Information Processing Systems}, isbn = {9781713845393}, location = {Virtual}, pages = {116--128}, title = {{Backward-compatible prediction updates: A probabilistic approach}}, volume = {34}, year = {2021}, } @unpublished{14221, abstract = {The world is structured in countless ways. It may be prudent to enforce corresponding structural properties to a learning algorithm's solution, such as incorporating prior beliefs, natural constraints, or causal structures. Doing so may translate to faster, more accurate, and more flexible models, which may directly relate to real-world impact. In this dissertation, we consider two different research areas that concern structuring a learning algorithm's solution: when the structure is known and when it has to be discovered.}, author = {Locatello, Francesco}, booktitle = {arXiv}, title = {{Enforcing and discovering structure in machine learning}}, doi = {10.48550/arXiv.2111.13693}, year = {2021}, } @unpublished{14278, abstract = {The Birkhoff conjecture says that the boundary of a strictly convex integrable billiard table is necessarily an ellipse. In this article, we consider a stronger notion of integrability, namely, integrability close to the boundary, and prove a local version of this conjecture: a small perturbation of almost every ellipse that preserves integrability near the boundary, is itself an ellipse. We apply this result to study local spectral rigidity of ellipses using the connection between the wave trace of the Laplacian and the dynamics near the boundary and establish rigidity for almost all of them.}, author = {Koval, Illya}, booktitle = {arXiv}, title = {{Local strong Birkhoff conjecture and local spectral rigidity of almost every ellipse}}, doi = {10.48550/ARXIV.2111.12171}, year = {2021}, } @inproceedings{14332, abstract = {Learning data representations that are useful for various downstream tasks is a cornerstone of artificial intelligence. While existing methods are typically evaluated on downstream tasks such as classification or generative image quality, we propose to assess representations through their usefulness in downstream control tasks, such as reaching or pushing objects. By training over 10,000 reinforcement learning policies, we extensively evaluate to what extent different representation properties affect out-of-distribution (OOD) generalization. Finally, we demonstrate zero-shot transfer of these policies from simulation to the real world, without any domain randomization or fine-tuning. This paper aims to establish the first systematic characterization of the usefulness of learned representations for real-world OOD downstream tasks.}, author = {Träuble, Frederik and Dittadi, Andrea and Wuthrich, Manuel and Widmaier, Felix and Gehler, Peter Vincent and Winther, Ole and Locatello, Francesco and Bachem, Olivier and Schölkopf, Bernhard and Bauer, Stefan}, booktitle = {ICML 2021 Workshop on Unsupervised Reinforcement Learning}, location = {Virtual}, title = {{Representation learning for out-of-distribution generalization in reinforcement learning}}, year = {2021}, } @article{14800, abstract = {Research on two-dimensional (2D) materials has been explosively increasing in last seventeen years in varying subjects including condensed matter physics, electronic engineering, materials science, and chemistry since the mechanical exfoliation of graphene in 2004. Starting from graphene, 2D materials now have become a big family with numerous members and diverse categories. The unique structural features and physicochemical properties of 2D materials make them one class of the most appealing candidates for a wide range of potential applications. In particular, we have seen some major breakthroughs made in the field of 2D materials in last five years not only in developing novel synthetic methods and exploring new structures/properties but also in identifying innovative applications and pushing forward commercialisation. In this review, we provide a critical summary on the recent progress made in the field of 2D materials with a particular focus on last five years. After a brief background introduction, we first discuss the major synthetic methods for 2D materials, including the mechanical exfoliation, liquid exfoliation, vapor phase deposition, and wet-chemical synthesis as well as phase engineering of 2D materials belonging to the field of phase engineering of nanomaterials (PEN). We then introduce the superconducting/optical/magnetic properties and chirality of 2D materials along with newly emerging magic angle 2D superlattices. Following that, the promising applications of 2D materials in electronics, optoelectronics, catalysis, energy storage, solar cells, biomedicine, sensors, environments, etc. are described sequentially. Thereafter, we present the theoretic calculations and simulations of 2D materials. Finally, after concluding the current progress, we provide some personal discussions on the existing challenges and future outlooks in this rapidly developing field. }, author = {Chang, Cheng and Chen, Wei and Chen, Ye and Chen, Yonghua and Chen, Yu and Ding, Feng and Fan, Chunhai and Fan, Hong Jin and Fan, Zhanxi and Gong, Cheng and Gong, Yongji and He, Qiyuan and Hong, Xun and Hu, Sheng and Hu, Weida and Huang, Wei and Huang, Yuan and Ji, Wei and Li, Dehui and Li, Lain Jong and Li, Qiang and Lin, Li and Ling, Chongyi and Liu, Minghua and Liu, Nan and Liu, Zhuang and Loh, Kian Ping and Ma, Jianmin and Miao, Feng and Peng, Hailin and Shao, Mingfei and Song, Li and Su, Shao and Sun, Shuo and Tan, Chaoliang and Tang, Zhiyong and Wang, Dingsheng and Wang, Huan and Wang, Jinlan and Wang, Xin and Wang, Xinran and Wee, Andrew T.S. and Wei, Zhongming and Wu, Yuen and Wu, Zhong Shuai and Xiong, Jie and Xiong, Qihua and Xu, Weigao and Yin, Peng and Zeng, Haibo and Zeng, Zhiyuan and Zhai, Tianyou and Zhang, Han and Zhang, Hui and Zhang, Qichun and Zhang, Tierui and Zhang, Xiang and Zhao, Li Dong and Zhao, Meiting and Zhao, Weijie and Zhao, Yunxuan and Zhou, Kai Ge and Zhou, Xing and Zhou, Yu and Zhu, Hongwei and Zhang, Hua and Liu, Zhongfan}, issn = {1001-4861}, journal = {Acta Physico-Chimica Sinica}, number = {12}, publisher = {Peking University}, title = {{Recent progress on two-dimensional materials}}, doi = {10.3866/PKU.WHXB202108017}, volume = {37}, year = {2021}, } @article{14889, abstract = {We consider the Fröhlich Hamiltonian with large coupling constant α. For initial data of Pekar product form with coherent phonon field and with the electron minimizing the corresponding energy, we provide a norm approximation of the evolution, valid up to times of order α2. The approximation is given in terms of a Pekar product state, evolved through the Landau-Pekar equations, corrected by a Bogoliubov dynamics taking quantum fluctuations into account. This allows us to show that the Landau-Pekar equations approximately describe the evolution of the electron- and one-phonon reduced density matrices under the Fröhlich dynamics up to times of order α2.}, author = {Leopold, Nikolai K and Mitrouskas, David Johannes and Rademacher, Simone Anna Elvira and Schlein, Benjamin and Seiringer, Robert}, issn = {2578-5885}, journal = {Pure and Applied Analysis}, number = {4}, pages = {653--676}, publisher = {Mathematical Sciences Publishers}, title = {{Landau–Pekar equations and quantum fluctuations for the dynamics of a strongly coupled polaron}}, doi = {10.2140/paa.2021.3.653}, volume = {3}, year = {2021}, } @article{14890, abstract = {We consider a system of N interacting bosons in the mean-field scaling regime and construct corrections to the Bogoliubov dynamics that approximate the true N-body dynamics in norm to arbitrary precision. The N-independent corrections are given in terms of the solutions of the Bogoliubov and Hartree equations and satisfy a generalized form of Wick's theorem. We determine the n-point correlation functions of the excitations around the condensate, as well as the reduced densities of the N-body system, to arbitrary accuracy, given only the knowledge of the two-point functions of a quasi-free state and the solution of the Hartree equation. In this way, the complex problem of computing all n-point correlation functions for an interacting N-body system is essentially reduced to the problem of solving the Hartree equation and the PDEs for the Bogoliubov two-point functions.}, author = {Bossmann, Lea and Petrat, Sören P and Pickl, Peter and Soffer, Avy}, issn = {2578-5885}, journal = {Pure and Applied Analysis}, number = {4}, pages = {677--726}, publisher = {Mathematical Sciences Publishers}, title = {{Beyond Bogoliubov dynamics}}, doi = {10.2140/paa.2021.3.677}, volume = {3}, year = {2021}, } @inbook{14984, abstract = {Hybrid zones are narrow geographic regions where different populations, races or interbreeding species meet and mate, producing mixed ‘hybrid’ offspring. They are relatively common and can be found in a diverse range of organisms and environments. The study of hybrid zones has played an important role in our understanding of the origin of species, with hybrid zones having been described as ‘natural laboratories’. This is because they allow us to study,in situ, the conditions and evolutionary forces that enable divergent taxa to remain distinct despite some ongoing gene exchange between them.}, author = {Stankowski, Sean and Shipilina, Daria and Westram, Anja M}, booktitle = {Encyclopedia of Life Sciences}, isbn = {9780470016176}, publisher = {Wiley}, title = {{Hybrid Zones}}, doi = {10.1002/9780470015902.a0029355}, volume = {2}, year = {2021}, } @inbook{14987, abstract = {The goal of zero-shot learning is to construct a classifier that can identify object classes for which no training examples are available. When training data for some of the object classes is available but not for others, the name generalized zero-shot learning is commonly used. In a wider sense, the phrase zero-shot is also used to describe other machine learning-based approaches that require no training data from the problem of interest, such as zero-shot action recognition or zero-shot machine translation.}, author = {Lampert, Christoph}, booktitle = {Computer Vision}, editor = {Ikeuchi, Katsushi}, isbn = {9783030634155}, pages = {1395--1397}, publisher = {Springer}, title = {{Zero-Shot Learning}}, doi = {10.1007/978-3-030-63416-2_874}, year = {2021}, } @misc{14988, abstract = {Raw data generated from the publication - The TPLATE complex mediates membrane bending during plant clathrin-mediated endocytosis by Johnson et al., 2021 In PNAS}, author = {Johnson, Alexander J}, publisher = {Zenodo}, title = {{Raw data from Johnson et al, PNAS, 2021}}, doi = {10.5281/ZENODO.5747100}, year = {2021}, }