Project

General

Profile

Cleanup #512

Remove duplication of conv-mach files

Added by Lukasz Wesolowski about 5 years ago. Updated over 2 years ago.

Status:
In Progress
Priority:
Low
Assignee:
PPL
Category:
-
Target version:
-
Start date:
08/15/2014
Due date:
08/31/2014
% Done:

0%

Estimated time:
10.00 h
Spent time:

Description

Many conv-mach files are completely identical for different builds of charm. Wherever possible, the duplication should be removed.

History

#1 Updated by Lukasz Wesolowski over 4 years ago

Obtained a list of duplicates in src/arch using the fdupes utility in linux. I am pasting the results below. Each group indicates duplicates. The last group, which is the largest, consists of empty files, the need for which is unpleasant in itself. I will think about how to remove the duplication. Adding symbolic links or placing common files in parent directories are the likely options.

./gni-crayxe/special.sh
./gni-crayxc/special.sh

./net-linux-x86_64/conv-mach-sysvshm.h
./net-linux-arm7/conv-mach-sysvshm.h
./netlrts-linux-x86_64/conv-mach-sysvshm.h

./net-linux-x86_64/conv-mach-smp.h
./net-linux-arm7/conv-mach-smp.h
./netlrts-linux-x86_64/conv-mach-smp.h
./verbs-linux-x86_64/conv-mach-smp.h

./net-linux-x86_64/conv-mach-clustermatic.sh
./netlrts-linux-x86_64/conv-mach-clustermatic.sh
./verbs-linux-x86_64/conv-mach-clustermatic.sh

./net-linux-x86_64/conv-mach-pedantic.sh
./net-linux-arm7/conv-mach-pedantic.sh
./netlrts-linux-x86_64/conv-mach-pedantic.sh
./verbs-linux-x86_64/conv-mach-pedantic.sh

./net-linux-x86_64/conv-mach-ibverbs.sh
./net-linux-x86_64/conv-mach-ibud.sh
./net-linux-arm7/conv-mach-ibverbs.sh
./verbs-linux-x86_64/conv-mach-ibud.sh

./net/conv-mach-local.sh
./netlrts/conv-mach-local.sh
./verbs/conv-mach-local.sh

./net/conv-mach-tcp.sh
./netlrts/conv-mach-tcp.sh

./net/conv-mach-causalft.h
./netlrts/conv-mach-causalft.h
./verbs/conv-mach-causalft.h

./net/machine-recover.c
./netlrts/machine-recover.c
./verbs/machine-recover.c

./net/conv-mach-local.h
./netlrts/conv-mach-local.h
./verbs/conv-mach-local.h

./common/cc-icc8.h
./common/cc-iccstatic.h
./common/cc-icc.h

./common/conv-mach-bigemulator.sh
./common/conv-mach-bigsim.sh

./net-linux/conv-mach-smp.h
./netlrts-linux/conv-mach-smp.h

./net-linux/conv-mach-clustermatic.h
./netlrts-linux/conv-mach-clustermatic.h

./net-linux/conv-mach-clustermatic.sh
./netlrts-linux/conv-mach-clustermatic.sh

./net-linux/conv-mach-turing.h
./netlrts-linux/conv-mach-turing.h

./net-linux/conv-mach-scyld.sh
./netlrts-linux/conv-mach-scyld.sh

./net-linux/conv-mach-turing.sh
./netlrts-linux/conv-mach-turing.sh

./net-linux/conv-mach-scyld.h
./net-linux-x86_64/conv-mach-clustermatic.h
./netlrts-linux-x86_64/conv-mach-clustermatic.h
./netlrts-linux/conv-mach-scyld.h
./verbs-linux-x86_64/conv-mach-clustermatic.h

./gni-crayxc/conv-mach-smp.h
./gni-crayxe/conv-mach-smp.h

./gni-crayxc/charmrun
./gni-crayxe/charmrun

./gni-crayxc/conv-mach-xpmem.sh
./gni-crayxe/conv-mach-xpmem.sh

./gni-crayxc/conv-mach-pxshm.h
./gni-crayxe/conv-mach-pxshm.h

./gni-crayxc/conv-mach-pxshm.sh
./gni-crayxe/conv-mach-pxshm.sh

./gni-crayxc/conv-mach-smp.sh
./gni-crayxe/conv-mach-smp.sh

./gni-crayxc/conv-mach-xpmem.h
./gni-crayxe/conv-mach-xpmem.h

./mpi-linux-mips64/conv-mach-smp.h
./mpi-sol/conv-mach-smp.h
./mpi-linux/conv-mach-smp.h

./net-darwin-x86_64/conv-mach-smp.h
./netlrts-darwin-x86_64/conv-mach-smp.h

./net-darwin-x86_64/conv-mach-smp.sh
./netlrts-darwin-x86_64/conv-mach-smp.sh
./mpi-darwin-x86_64/conv-mach-smp.sh

./net-darwin-x86_64/conv-mach.sh
./netlrts-darwin-x86_64/conv-mach.sh

./pami/Makefile.machine
./pamilrts/Makefile.machine

./net-linux-ppc/cc-xlc.h
./multicore-linux-ppc/cc-xlc.h

./net-linux-ppc/cc-xlc64.h
./multicore-linux-ppc/cc-xlc64.h

./net-linux-ppc/cc-xlc.sh
./multicore-linux-ppc/cc-xlc.sh

./mpi-linux-x86_64/conv-mach-gm2.h
./mpi-linux/conv-mach-gm2.h

./mpi-linux-x86_64/special.sh
./mpi-crayxe/special.sh
./net-linux-x86_64/special.sh
./net-linux-arm7/special.sh
./netlrts-linux-x86_64/special.sh
./verbs-linux-x86_64/special.sh
./multicore-arm7/special.sh

./mpi-linux-x86_64/conv-mach-scyld.sh
./mpi-linux/conv-mach-scyld.sh

./mpi-linux-x86_64/conv-mach-cuda.h
./mpi-crayxc/conv-mach-cuda.h
./mpi-crayxe/conv-mach-cuda.h
./net-linux-x86_64/conv-mach-cuda.h
./gni-crayxe/conv-mach-cuda.h
./net-linux-arm7/conv-mach-cuda.h
./netlrts-linux-x86_64/conv-mach-cuda.h
./multicore-arm7/conv-mach-cuda.h
./gni-crayxc/conv-mach-cuda.h

./mpi-linux-x86_64/conv-mach-cuda.sh
./mpi-crayxc/conv-mach-cuda.sh
./mpi-crayxe/conv-mach-cuda.sh
./net-linux-x86_64/conv-mach-cuda.sh
./gni-crayxe/conv-mach-cuda.sh
./net-linux-arm7/conv-mach-cuda.sh
./netlrts-linux-x86_64/conv-mach-cuda.sh
./verbs-linux-x86_64/conv-mach-cuda.sh
./multicore-arm7/conv-mach-cuda.sh
./gni-crayxc/conv-mach-cuda.sh

./win32/unix2nt_ar.old
./win64/unix2nt_ar.old

./win32/unix2nt_cc.old
./win64/unix2nt_cc.old

./net-sol/conv-mach-smp.sh
./mpi-linux-x86_64/conv-mach-smp.sh
./mpi-linux-mips64/conv-mach-smp.sh
./mpi-sol/conv-mach-smp.sh
./mpi-linux/conv-mach-smp.sh

./net-sol-x86_64/cc-cc.h
./net-sol/cc-cc.h

./net-sol-x86_64/cc-cc64.h
./net-sol/cc-cc64.h

./mpi-bluegenep/conv-mach-smp.h
./bluegenep/conv-mach-smp.h

./pamilrts-bluegeneq/conv-mach.h
./pami-bluegeneq/conv-mach.h

./pamilrts-bluegeneq/cc-gcc.sh
./pami-bluegeneq/cc-gcc.sh

./netlrts-linux/conv-mach-sysvshm.h
./net-linux/conv-mach-sysvshm.h

./netlrts-linux/conv-mach.h
./net-linux/conv-mach.h

./netlrts-linux/conv-mach-pxshm.h
./net-linux/conv-mach-pxshm.h

./netlrts-linux/conv-mach-smp.sh
./net-linux/conv-mach-smp.sh

./netlrts-linux/conv-mach.sh
./net-linux/conv-mach.sh

./mpi/conv-mach-pthreads.h
./net-linux-arm7/conv-mach-pthreads.h

./mpi/conv-mach-pthreads.sh
./net-linux-arm7/conv-mach-pthreads.sh

./mpi/conv-mach-mlogft.h
./gni/conv-mach-mlogft.h
./net/conv-mach-mlogft.h
./netlrts/conv-mach-mlogft.h
./verbs/conv-mach-mlogft.h

./mpi-crayxt/conv-mach-smp.h
./mpi-crayxc/conv-mach-smp.h
./mpi-crayxe/conv-mach-smp.h

./mpi-crayxt/conv-mach-smp.sh
./net-linux-ppc/conv-mach-smp.sh
./mpi-crayxc/conv-mach-smp.sh
./mpi-crayxe/conv-mach-smp.sh
./net-linux-x86_64/conv-mach-smp.sh
./net-linux-arm7/conv-mach-smp.sh
./netlrts-linux-x86_64/conv-mach-smp.sh
./verbs-linux-x86_64/conv-mach-smp.sh

./multicore-win64/conv-mach.sh
./multicore-win32/conv-mach.sh

./mpi-bluegeneq/cc-xlc.h
./mpi-bluegenep/cc-xlc.h
./bluegenep/cc-xlc64.h
./bluegenep/cc-xlc.h

./mpi-bluegeneq/charmrun
./pami-bluegeneq/charmrun

./mpi-bluegeneq/conv-mach-smp.sh
./pamilrts-bluegeneq/conv-mach-smp.sh
./pami-bluegeneq/conv-mach-smp.sh

./netlrts-linux-x86_64/conv-mach.h
./net-linux-x86_64/conv-mach.h

./netlrts-linux-x86_64/conv-mach.sh
./net-linux-x86_64/conv-mach.sh

./netlrts-darwin-x86_64/conv-mach.h
./net-darwin-x86_64/conv-mach.h

./net-aix-ppc/cc-gcc.h
./common/cc-clang.h
./common/cc-gcc.h

./net-aix-ppc/conv-mach-pxshm.sh
./net-linux/conv-mach-pxshm.sh
./net-linux-x86_64/conv-mach-pxshm.sh
./net-linux-arm7/conv-mach-pxshm.sh
./netlrts-linux-x86_64/conv-mach-pxshm.sh
./netlrts-linux/conv-mach-pxshm.sh
./verbs-linux-x86_64/conv-mach-pxshm.sh

./net-darwin-x86/conv-mach-pxshm.h
./net-darwin-x86_64/conv-mach-pxshm.h
./net-linux-x86_64/conv-mach-pxshm.h
./net-linux-arm7/conv-mach-pxshm.h
./netlrts-darwin-x86_64/conv-mach-pxshm.h
./netlrts-linux-x86_64/conv-mach-pxshm.h

./net-darwin-x86/conv-mach-pxshm.sh
./net-darwin-x86_64/conv-mach-pxshm.sh
./net-linux/conv-mach-sysvshm.sh
./net-linux-x86_64/conv-mach-sysvshm.sh
./net-linux-arm7/conv-mach-sysvshm.sh
./netlrts-darwin-x86_64/conv-mach-pxshm.sh
./netlrts-linux-x86_64/conv-mach-sysvshm.sh
./netlrts-linux/conv-mach-sysvshm.sh
./mpi-darwin-x86_64/conv-mach-pxshm.sh

./shmem-crayxt/charmrun
./shmem-crayxe/charmrun

./net-cygwin/conv-mach-smp.h
./net-win64/conv-mach-smp.h
./mpi-cygwin/conv-mach-smp.h

./net-cygwin/conv-mach-smp.sh
./net-win64/conv-mach-smp.sh
./mpi/conv-mach-lam.h
./mpi/conv-mach-syncft.sh
./mpi/conv-mach-mlogft.sh
./mpi/conv-mach-gm.sh
./mpi/conv-mach-causalft.sh
./mpi-bluegenep/conv-mach-smp.sh
./mpi-bluegenep/cc-mpcc.h
./mpi-linux-x86_64/conv-mach-scyld.h
./gni/conv-mach-syncft.sh
./gni/conv-mach-mlogft.sh
./pami/conv-mach-async.sh
./pami/conv-common.sh
./mpi-cygwin/conv-mach-smp.sh
./net-win32/conv-mach-smp.sh
./bluegenep/conv-mach-smp.sh
./mpi-sp/conv-mach-uP.h
./common/conv-mach-ifort.h
./common/conv-mach-lcs.h
./common/conv-mach-ooc.sh
./common/conv-mach-pgf90.h
./net/conv-mach-syncft.sh
./net/conv-mach-mlogft.sh
./net/conv-mach-causalft.sh
./pamilrts/conv-mach-async.sh
./pamilrts/conv-common.sh
./mpi-linux/conv-mach-scyld.h
./mpi-linux/conv-mach-gm.sh
./mpi-linux/conv-mach-gm2.sh
./net-linux-x86_64/conv-mach-pedantic.h
./net-linux-arm7/conv-mach-pedantic.h
./netlrts/conv-mach-syncft.sh
./netlrts/conv-mach-mlogft.sh
./netlrts/conv-mach-causalft.sh
./netlrts-linux-x86_64/conv-mach-pedantic.h
./mpi-bluegeneq/cc-gcc.h
./verbs/conv-mach-syncft.sh
./verbs/conv-mach-mlogft.sh
./verbs/conv-mach-causalft.sh
./pamilrts-bluegeneq/cc-xlc.sh
./pamilrts-bluegeneq/cc-gcc.h
./gni/conv-mach-hugepages.h
./common/cc-craycc.h
./pami-bluegeneq/cc-xlc.sh
./pami-bluegeneq/cc-gcc.h
./verbs-linux-x86_64/conv-mach-pedantic.h
./gni/conv-mach-papi.h
./common/conv-mach-tsan.h

#2 Updated by Lukasz Wesolowski over 4 years ago

  • Status changed from New to In Progress

#3 Updated by Phil Miller about 4 years ago

  • Assignee changed from Lukasz Wesolowski to PPL

#4 Updated by Michael Robson almost 4 years ago

  • Assignee changed from PPL to Michael Robson

#5 Updated by Michael Robson over 3 years ago

I fixed the special.sh duplication so there is only:

./common/special.sh (for CUDA builds)
./net-linux-cell/special.sh (for cell builds)

The latter should overwrite the former when making cell

#6 Updated by Michael Robson over 2 years ago

  • Assignee changed from Michael Robson to PPL
  • Priority changed from Normal to Low
I re-ran fdupes on current charm and generated the following list. Since net and pami will (one day) be deleted, I removed most of the entries pertaining to them. I then tried to group the remaining lines to get an idea of common files. I was able to abstract the following:
  • gni
  • cuda
  • mlogft
  • syncft
  • causalft
In the future, people might want to look at :
  • netlrs/verbs similarities
  • smp abstraction (seems like lots of layers are very similar but not identical)

I also confirmed that more specific layers (ie mpi-linux-x86_64) overwrites the less specific (network) layers (ie mpi). Here's the current list, grouped, with strike-throughs indicating those duplicates that have been removed:

gni
./gni-crayxe/conv-mach-smp.h
./gni-crayxc/conv-mach-smp.h

./gni-crayxe/conv-mach-pxshm.sh
./gni-crayxc/conv-mach-pxshm.sh

./gni-crayxe/conv-mach-pxshm.h
./gni-crayxc/conv-mach-pxshm.h

./gni-crayxe/conv-mach-xpmem.sh
./gni-crayxc/conv-mach-xpmem.sh

./gni-crayxe/conv-mach-xpmem.h
./gni-crayxc/conv-mach-xpmem.h

./gni-crayxe/conv-mach-smp.sh
./gni-crayxc/conv-mach-smp.sh

cuda
./gni-crayxe/conv-mach-cuda.sh
./mpi-crayxc/conv-mach-cuda.sh
./mpi-crayxe/conv-mach-cuda.sh
./mpi-linux-x86_64/conv-mach-cuda.sh
./net-linux-arm7/conv-mach-cuda.sh
./net-linux-x86_64/conv-mach-cuda.sh
./netlrts-linux-x86_64/conv-mach-cuda.sh
./verbs-linux-x86_64/conv-mach-cuda.sh
./multicore-arm7/conv-mach-cuda.sh
./gni-crayxc/conv-mach-cuda.sh

./gni-crayxe/conv-mach-cuda.h
./mpi-crayxc/conv-mach-cuda.h
./mpi-crayxe/conv-mach-cuda.h
./mpi-linux-x86_64/conv-mach-cuda.h
./net-linux-arm7/conv-mach-cuda.h
./net-linux-x86_64/conv-mach-cuda.h
./netlrts-linux-x86_64/conv-mach-cuda.h
./multicore-arm7/conv-mach-cuda.h
./gni-crayxc/conv-mach-cuda.h
./multicore-linux64/conv-mach-cuda.h

mlogft
./gni/conv-mach-mlogft.h
./mpi/conv-mach-mlogft.h
./net/conv-mach-mlogft.h
./netlrts/conv-mach-mlogft.h
./verbs/conv-mach-mlogft.h

./gni/conv-mach-mlogft.sh
./mpi/conv-mach-mlogft.sh
./net/conv-mach-mlogft.sh
./netlrts/conv-mach-mlogft.sh
./verbs/conv-mach-mlogft.sh

causalft
./net/conv-mach-causalft.h
./netlrts/conv-mach-causalft.h
./verbs/conv-mach-causalft.h
(./mpi/conv-mach-causalft.h)

scyld
./mpi-linux-x86_64/conv-mach-scyld.sh
./mpi-linux/conv-mach-scyld.sh

gm2
./mpi-linux-x86_64/conv-mach-gm2.h
./mpi-linux/conv-mach-gm2.h

smp
  • darwin
    ./mpi-darwin-x86_64/conv-mach-smp.sh
    ./netlrts-darwin-x86_64/conv-mach-smp.sh
  • netlrts/verbs
    ./netlrts-linux-x86_64/conv-mach-smp.h
    ./verbs-linux-x86_64/conv-mach-smp.h

./netlrts-linux-x86_64/conv-mach-smp.sh
./verbs-linux-x86_64/conv-mach-smp.sh

  • mpi
    ./mpi-crayxc/conv-mach-smp.h
    ./mpi-crayxe/conv-mach-smp.h

./mpi-crayxe/conv-mach-smp.sh
./mpi-crayxc/conv-mach-smp.sh

./mpi-linux-mips64/conv-mach-smp.h
./mpi-linux/conv-mach-smp.h
./mpi-sol/conv-mach-smp.h

./mpi-linux-x86_64/conv-mach-smp.sh
./mpi-linux-mips64/conv-mach-smp.sh
./mpi-linux/conv-mach-smp.sh
./mpi-sol/conv-mach-smp.sh

win
./win64/unix2nt_cc
./win/unix2nt_cc

./win64/unix2nt_ar
./win/unix2nt_ar

./win32/unix2nt_cc.old
./win64/unix2nt_cc.old

./win32/unix2nt_ar.old
./win64/unix2nt_ar.old

common
./common/cc-icc.h
./common/cc-icc8.h
./common/cc-iccstatic.h

./common/cc-clang.h
./common/cc-gcc.h

./common/conv-mach-bigemulator.sh
./common/conv-mach-bigsim.sh

ppc
./multicore-linux-ppc/cc-xlc.h
./pami-linux-ppc64le/cc-gcc.h

netlrts/verbs
./netlrts/conv-mach-local.h
./verbs/conv-mach-local.h

./netlrts/conv-mach-local.sh
./verbs/conv-mach-local.sh

./netlrts/machine-recover.c
./verbs/machine-recover.c

./netlrts-linux-x86_64/conv-mach-clustermatic.sh
./verbs-linux-x86_64/conv-mach-clustermatic.sh

./netlrts-linux/conv-mach-scyld.h
./netlrts-linux-x86_64/conv-mach-clustermatic.h
./verbs-linux-x86_64/conv-mach-clustermatic.h

./netlrts-linux-x86_64/conv-mach-pxshm.sh
./netlrts-linux/conv-mach-pxshm.sh
./verbs-linux-x86_64/conv-mach-pxshm.sh

./netlrts-linux-x86_64/conv-mach-pedantic.sh
./verbs-linux-x86_64/conv-mach-pedantic.sh

./netlrts-linux-x86_64/conv-mach-sysvshm.sh
./netlrts-linux/conv-mach-sysvshm.sh
./netlrts-darwin-x86_64/conv-mach-pxshm.sh

empty
  • scyld
    ./mpi-linux-x86_64/conv-mach-scyld.h
    ./mpi-linux/conv-mach-scyld.h
  • causalft
    ./mpi/conv-mach-causalft.sh
    ./net/conv-mach-causalft.sh
    ./netlrts/conv-mach-causalft.sh
    ./verbs/conv-mach-causalft.sh
  • syncft
    ./gni/conv-mach-syncft.sh
    ./mpi/conv-mach-syncft.sh
    ./net/conv-mach-syncft.sh
    ./netlrts/conv-mach-syncft.sh
    ./verbs/conv-mach-syncft.sh
  • gm
    ./mpi/conv-mach-gm.sh
    ./mpi-linux/conv-mach-gm.sh
  • smp
    ./mpi-cygwin/conv-mach-smp.sh
    ./mpi-win64/conv-mach-smp.sh
    ./net-cygwin/conv-mach-smp.sh
    ./net-win64/conv-mach-smp.sh
    ./netlrts-win-x86_64/conv-mach-smp.sh
  • pedantic
    ./netlrts-linux-x86_64/conv-mach-pedantic.h
    ./verbs-linux-x86_64/conv-mach-pedantic.h

./common/conv-mach-ooc.sh
./common/conv-mach-pgf90.h
./common/conv-mach-lcs.h
./common/conv-mach-ifort.h

./gni/conv-mach-hugepages.h
./gni/conv-mach-papi.h

./mpi/conv-mach-lam.h
./mpi-bluegeneq/cc-gcc.h
./mpi-linux/conv-mach-gm2.sh

./netlrts/conv-mach-tcp.sh

Also available in: Atom PDF