*** Settings ***
Library			  Collections
#Library           RequestsLibrary
Library           Process
#Library           SSHLibrary
Library           Resources/keywords/fs_fault_inject.py
Library           Resources/keywords/fault_inject.py
Library           Resources/keywords/deploy.py

Suite Setup       init failover cluster
Suite Teardown    clean failover env

*** Variables ***
${fio_vdsize}     10
${vdbench_vdsize}   10
${vm_iops_limit}   8000
${chunk_num}   0

*** Test Cases ***
begin pjdtest
    [Tags]   P2  base    interface    pjdtest
    ${pjdfs}    evaluate    str("failover-pjdtest")
    mount test dir   ${pjdfs}
    start pjdtest    ${pjdfs}

inject kill one mds
    [Tags]    P0    base    first release   failover
    ${num}   evaluate  int(1)
    test kill process   mds   ${num}
    check fuse iops
    sleep  3
    test start process  mds
    check fuse mount success
    check fuse iops
    check fs cluster ok
    [Teardown]    test start process  mds

inject kill one metaserver
    [Tags]    P0    base    first release   failover
    ${num}   evaluate  int(1)
    test kill process   metaserver   ${num}
    check fuse iops
    sleep  3
    test start process  metaserver
    check fuse mount success
    check fuse iops
    check fs cluster ok
    [Teardown]    test start process  metaserver

inject kill one etcd
    [Tags]    P0    base    first release   failover
    ${num}   evaluate  int(1)
    test kill process   etcd   ${num}
    check fuse iops
    sleep  3
    test start process  etcd
    check fuse mount success
    check fuse iops
    check fs cluster ok
    [Teardown]    test start process  etcd

test fs mds loss package 5%
    [Tags]    P1    base    first release   failover  
    ${percent}  evaluate  int(5)
    test fs process loss package   mds   ${percent}
    check fuse mount success
    check fuse iops
    [Teardown]	wait fs cluster ok 

test fs metaserver loss package 5%
    [Tags]    P1    base    first release   failover
    ${percent}  evaluate  int(5)
    test fs process loss package  metaserver  ${percent}
    check fuse mount success
    check fuse iops
    [Teardown]  check fs cluster ok

test fs client loss package 5%
    [Tags]    P1    base    first release   failover
    ${percent}  evaluate  int(5)
    test fs process loss package  fuseclient  ${percent}
    check fuse mount success
    check fuse iops
    [Teardown]  check fs cluster ok

test fs etcd loss package 5%
    [Tags]    P1    base    first release   failover
    ${percent}  evaluate  int(5)
    test fs process loss package  etcd   ${percent}
    check fuse mount success
    check fuse iops
    [Teardown]  check fs cluster ok

test fs mds delay package 100ms
    [Tags]    P1    base    first release   failover
    ${ms}  evaluate  int(100)
    test fs process delay package  mds  ${ms}
    check fuse mount success
    check fuse iops
    [Teardown]  check fs cluster ok

test fs metaserver delay package 100ms
    [Tags]    P1    base    first release   failover
    ${ms}  evaluate  int(100)
    test fs process delay package  metaserver  ${ms}
    check fuse mount success
    check fuse iops
    [Teardown]  check fs cluster ok

test fs etcd delay package 100ms
    [Tags]    P1    base    first release   failover
    ${ms}  evaluate  int(100)
    test fs process delay package  etcd  ${ms}
    check fuse mount success
    check fuse iops
    [Teardown]  check fs cluster ok

test fs client delay package 100ms
    [Tags]    P1    base    first release   failover
    ${ms}  evaluate  int(100)
    test fs process delay package  fuseclient  ${ms}
    check fuse mount success
    check fuse iops
    [Teardown]  check fs cluster ok

inject fuse client cpu stress 95%
    [Tags]    P1    base    first release   failover
    ${cpu_stress}   evaluate  int(95)
    ${ssh}   test fuse client cpu stress   ${cpu_stress}
    sleep  5
    del cpu stress   ${ssh}
    check fuse mount success
    check fuse iops
    [Teardown]     del cpu stress    ${ssh}

inject fuse client mem stress 90%
    [Tags]    P1    base    first release   failover
    ${mem_stress}   evaluate  int(75)
    ${ssh}   test fuse client mem stress   ${mem_stress}
    sleep  5
    del mem stress   ${ssh}
    check fuse mount success
    check fuse iops
    [Teardown]  wait fs cluster ok

test metaserver copyset migrates out in
    [Tags]    P1    base    first release   failover
    test out metaserver copyset
    test in metaserver copyset
    check fuse mount success

test ipmitool restart metaserver node
   [Tags]    P1    base    first release   failover  ipmitool
   test ipmitool restart metaserver
   sleep  60
   check fuse mount success
   check fuse iops
   [Teardown]   wait fs cluster ok

test check pjdtest result
    [Tags]   P2  base    interface    pjdtest
    check pjdtest result

test loop mount umount
    [Tags]    P0    base    first release   failover
    sleep   30
    stop loop mount
    check fuse mount success

inject kill two mds
    [Tags]    P1    base    first release   failover
    ${num}   evaluate  int(2)
    test kill process   mds   ${num}
    sleep  5
    test start process  mds
    sleep  5
    check fuse mount success
    check fuse iops
    check fs cluster ok
    [Teardown]    test start process  mds

inject kill all mds
    [Tags]    P1    base    first release   failover
    ${num}   evaluate  int(3)
    test kill process   mds   ${num}
    sleep  5
    test start process  mds
    sleep  5
    check fuse mount success
    check fuse iops
    check fs cluster ok
    [Teardown]    wait fs cluster ok

inject kill two metaserver
    [Tags]    P1    base    first release   failover
    ${num}   evaluate  int(2)
    test kill process   metaserver   ${num}
    sleep  5
    test start process  metaserver
    sleep  5
    check fuse mount success
    check fuse iops
    check fs cluster ok
    [Teardown]    wait fs cluster ok

inject kill all metaserver
    [Tags]    P1    base    first release   failover
    ${num}   evaluate  int(3)
    test kill process   metaserver   ${num}
    sleep  5
    test start process  metaserver
    sleep  5
    check fuse mount success
    check fuse iops
    check fs cluster ok
    [Teardown]   wait fs cluster ok

inject kill two etcd
    [Tags]    P1    base    first release   failover
    ${num}   evaluate  int(2)
    test kill process   etcd   ${num}
    sleep  5
    test start process  etcd
    sleep  5
    check fuse mount success
    check fuse iops
    check fs cluster ok
    [Teardown]    test start process  etcd

inject kill all etcd
    [Tags]    P1    base    first release   failover
    ${num}   evaluate  int(3)
    test kill process   etcd   ${num}
    sleep  5
    test start process  etcd
    sleep  5
    check fuse mount success
    check fuse iops
    check fs cluster ok
    [Teardown]    test start process  etcd

check cp data consistency
    [Tags]    P0    base    first release   failover  
    checksum data
    check fuse mount success
    wait op finish

check mv data consistency
    [Tags]    P1    base    first release   failover 
    mv data
    checksum mv data
    check fuse mount success

check cto file md5
    [Tags]    P0    base    first release   failover
    stop cto file md5check

check umount date consistency
    [Tags]    P1    base    first release   failover
    sleep  60
    get test dir file md5
    umount test dir   test2
    sleep  120
    wait fuse exit   failover/test2
    mount test dir   test2
    sleep  60
    check test dir file md5
    check cto dir file md5  test5
#    [Teardown]   wait fs cluster ok

*** Keywords ***

init failover cluster
     clean env
     destroy curvefs
     clean log
     deploy all servers
     sleep  30
     check fs cluster ok
     mount test dir
     sleep  40
     check fuse mount success
     write check data
     cp check data
     start fs op test
     mount fuse cto test 

clean log
     clean fs kernel log
     clean corefile

start fs op test
     start fs vdbench
     start fs fio
     loop mount umount
     sleep  60

mount fuse cto test 
     mount test dir   test1  test4
     mount test dir   test2  test5
     start cto file md5check
     
clean failover env
     check log
#     destroy curvefs

check log
     check fs io error
     check vdbench output
     check corefile

